From d16af9df59af228f321cc6dd76d90c641ab8ed8f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:23:57 -0500 Subject: [PATCH 01/10] USHIFT-936: align the DNS portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 74 +++++++++++++++---- pkg/cmd/init.go | 8 +- pkg/cmd/run.go | 2 +- pkg/cmd/showConfig.go | 4 +- pkg/components/render.go | 2 +- pkg/config/config.go | 11 +-- pkg/config/config_test.go | 8 +- pkg/controllers/kube-apiserver.go | 2 +- 8 files changed, 80 insertions(+), 31 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index cc2c3247fd..53be1d09c6 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/mitchellh/go-homedir" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/component-base/logs" "k8s.io/klog/v2" @@ -58,7 +59,7 @@ type IngressConfig struct { ServingKey []byte } -type EtcdConfig struct { +type InternalEtcdConfig struct { // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value QuotaBackendBytes int64 // If the backend is fragmented more than `maxFragmentedPercentage` @@ -71,6 +72,19 @@ type EtcdConfig struct { DoStartupDefrag bool } +type EtcdConfig struct { + // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value + QuotaBackendSize string + // If the backend is fragmented more than `maxFragmentedPercentage` + // and the database size is greater than `minDefragSize`, do a defrag. + MinDefragSize string + MaxFragmentedPercentage float64 + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). + DefragCheckFreq string + // Whether or not to do a defrag when the server finishes starting + DoStartupDefrag bool +} + type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` @@ -86,20 +100,22 @@ type MicroshiftConfig struct { // Determines if kube-apiserver controller should configure the // KASAdvertiseAddress in the loopback interface. Automatically computed. SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd InternalEtcdConfig `json:"etcd"` + + DNS DNS `json:"-"` } // Top level config file type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Debugging Debugging `json:"debugging"` + Etcd EtcdConfig `json:"etcd"` } type Network struct { @@ -231,14 +247,16 @@ func NewMicroshiftConfig() *MicroshiftConfig { SubjectAltNames: subjectAltNames, NodeName: nodeName, NodeIP: nodeIP, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.42.0.0/16", ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: EtcdConfig{ + Etcd: InternalEtcdConfig{ MinDefragBytes: 100 * 1024 * 1024, // 100MB MaxFragmentedPercentage: 45, // percent DefragCheckFreq: 5 * time.Minute, @@ -392,9 +410,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Network.ServiceNodePortRange != "" { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain - } + c.DNS = config.DNS if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } @@ -402,6 +418,36 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } + if config.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) + if err != nil { + return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) + } + c.Etcd.DefragCheckFreq = d + } + if config.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) + if err != nil { + return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) + } + if !q.IsZero() { + c.Etcd.MinDefragBytes = q.Value() + } + } + if config.Etcd.MaxFragmentedPercentage > 0 { + c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage + } + if config.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) + if err != nil { + return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) + } + if !q.IsZero() { + c.Etcd.QuotaBackendBytes = q.Value() + } + } + c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag + return nil } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 01db0bd4f9..2640023640 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -72,7 +72,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err externalCertNames := []string{ cfg.NodeName, - "api." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, } externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add @@ -229,7 +229,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, Hostnames: []string{ - "*.apps." + cfg.BaseDomain, // wildcard for any additional auto-generated domains + "*.apps." + cfg.DNS.BaseDomain, // wildcard for any additional auto-generated domains }, }, ), @@ -285,8 +285,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - "api." + cfg.BaseDomain, - "api-int." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, + "api-int." + cfg.DNS.BaseDomain, apiServerServiceIP.String(), }, }, diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index a7ccef6628..fc84d457d9 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -62,7 +62,7 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { cfg.Cluster.ServiceCIDR, ".svc", ".cluster.local", - "."+cfg.BaseDomain); err != nil { + "."+cfg.DNS.BaseDomain); err != nil { klog.Fatal(err) } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 21c57f5f08..1adfd5a7b0 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -55,9 +55,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: config.DNS{ - BaseDomain: cfg.BaseDomain, - }, + DNS: cfg.DNS, Node: config.Node{ HostnameOverride: cfg.NodeName, NodeIP: cfg.NodeIP, diff --git a/pkg/components/render.go b/pkg/components/render.go index ce2c26c39d..43c79e6478 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -28,7 +28,7 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar "ClusterCIDR": cfg.Cluster.ClusterCIDR, "ServiceCIDR": cfg.Cluster.ServiceCIDR, "ClusterDNS": cfg.Cluster.DNS, - "BaseDomain": cfg.BaseDomain, + "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { params[k] = v diff --git a/pkg/config/config.go b/pkg/config/config.go index c331890b61..91a7895197 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -100,11 +100,12 @@ type MicroshiftConfig struct { // Determines if kube-apiserver controller should configure the // KASAdvertiseAddress in the loopback interface. Automatically computed. SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` + + DNS DNS `json:"-"` } // Top level config file @@ -246,7 +247,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { SubjectAltNames: subjectAltNames, NodeName: strings.ToLower(nodeName), NodeIP: nodeIP, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.42.0.0/16", @@ -407,9 +410,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Network.ServiceNodePortRange != "" { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain - } + c.DNS = config.DNS if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1be50403c6..804dcecdd7 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -71,7 +71,9 @@ func TestConfigFile(t *testing.T) { NodeName: "node1", NodeIP: "1.2.3.4", KASAdvertiseAddress: "6.7.8.9", - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.20.30.40/16", @@ -172,7 +174,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { NodeIP: "1.2.3.4", KASAdvertiseAddress: "6.7.8.9", SkipKASInterface: true, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.20.30.40/16", diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index dc018c9f46..cdf6884c17 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -172,7 +172,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { APIVersion: "route.openshift.io/v1", Kind: "HostAssignmentAdmissionConfig", }, - Domain: "apps." + cfg.BaseDomain, + Domain: "apps." + cfg.DNS.BaseDomain, }, }, }, From 593aaf9e4629ce72d2f2ae27d28a76bcdf69e5bd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:40:43 -0500 Subject: [PATCH 02/10] USHIFT-936: align the Node portion of the config data structures --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 40 +++++++++---------- pkg/cmd/init.go | 16 ++++---- pkg/cmd/run.go | 4 +- pkg/cmd/showConfig.go | 11 ++--- pkg/components/render.go | 4 +- pkg/config/config.go | 40 +++++++++---------- pkg/config/config_test.go | 32 ++++++++------- pkg/loadbalancerservice/controller.go | 2 +- pkg/mdns/controller.go | 4 +- pkg/node/kubelet.go | 4 +- pkg/sysconfwatch/sysconfwatch_linux.go | 2 +- 12 files changed, 78 insertions(+), 85 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 451403a0ae..c59a915ea1 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -92,8 +92,8 @@ func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { s.etcdCfg.LCUrls = url2379 s.etcdCfg.ListenMetricsUrls = setURL([]string{"localhost"}, "2381") - s.etcdCfg.Name = cfg.NodeName - s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.NodeName, "localhost") + s.etcdCfg.Name = cfg.Node.HostnameOverride + s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.Node.HostnameOverride, "localhost") s.etcdCfg.CipherSuites = tlsCipherSuites s.etcdCfg.ClientTLSInfo.CertFile = cryptomaterial.PeerCertPath(etcdServingCertDir) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 53be1d09c6..8dfd211194 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -89,8 +89,6 @@ type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into // the IP configured in the endpoint slice for kubernetes service. Must be @@ -105,7 +103,8 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` } // Top level config file @@ -245,8 +244,10 @@ func NewMicroshiftConfig() *MicroshiftConfig { return &MicroshiftConfig{ LogVLevel: 2, SubjectAltNames: subjectAltNames, - NodeName: nodeName, - NodeIP: nodeIP, + Node: Node{ + HostnameOverride: nodeName, + NodeIP: nodeIP, + }, DNS: DNS{ BaseDomain: "example.com", }, @@ -272,7 +273,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.NodeName == hostname + return c.Node.HostnameOverride == hostname } // Read or set the NodeName that will be used for this MicroShift instance @@ -282,10 +283,10 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.NodeName, nil + return c.Node.HostnameOverride, nil } else if err != nil { return "", err } @@ -294,8 +295,8 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } establishedNodeName, err := c.establishNodeName() @@ -303,14 +304,14 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.NodeName { + if establishedNodeName != c.Node.HostnameOverride { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) + c.Node.HostnameOverride, establishedNodeName) } else { - c.NodeName = establishedNodeName + c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) } } @@ -395,12 +396,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = config.Node.HostnameOverride - } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP - } + c.Node = config.Node if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -504,10 +500,10 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 2640023640..92ad9e3694 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -71,7 +71,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } externalCertNames := []string{ - cfg.NodeName, + cfg.Node.HostnameOverride, "api." + cfg.DNS.BaseDomain, } externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) @@ -81,8 +81,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.NodeIP { - externalCertNames = append(externalCertNames, cfg.NodeIP) + if cfg.KASAdvertiseAddress != cfg.Node.NodeIP { + externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -172,7 +172,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, // userinfo per https://kubernetes.io/docs/reference/access-authn-authz/node/#overview - UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.NodeName, Groups: []string{"system:nodes"}}, + UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.Node.HostnameOverride, Groups: []string{"system:nodes"}}, }, ).WithServingCertificates( &certchains.ServingCertificateSigningRequestInfo{ @@ -180,7 +180,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err Name: "kubelet-server", ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, - Hostnames: []string{cfg.NodeName}, + Hostnames: []string{cfg.Node.HostnameOverride}, }, ), ), @@ -314,7 +314,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-peer:etcd-client", Groups: []string{"system:etcd-peers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, &certchains.PeerCertificateSigningRequestInfo{ CSRMeta: certchains.CSRMeta{ @@ -322,7 +322,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-server:etcd-client", Groups: []string{"system:etcd-servers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, ), ).WithCABundle( @@ -390,7 +390,7 @@ func initKubeconfigs( } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.NodeName, "localhost") { + for _, name := range append(cfg.SubjectAltNames, cfg.Node.HostnameOverride, "localhost") { u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index fc84d457d9..0a5212c1ea 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -56,8 +56,8 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { // see https://github.com/openshift/microshift/pull/471 if err := util.AddToNoProxyEnv( - cfg.NodeIP, - cfg.NodeName, + cfg.Node.NodeIP, + cfg.Node.HostnameOverride, cfg.Cluster.ClusterCIDR, cfg.Cluster.ServiceCIDR, ".svc", diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 1adfd5a7b0..dd0eb19bbb 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -31,8 +31,8 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command switch opts.Mode { case "default": - cfg.NodeIP = "" - cfg.NodeName = "" + cfg.Node.NodeIP = "" + cfg.Node.HostnameOverride = "" case "effective": // Load the current configuration if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { @@ -55,11 +55,8 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: cfg.DNS, - Node: config.Node{ - HostnameOverride: cfg.NodeName, - NodeIP: cfg.NodeIP, - }, + DNS: cfg.DNS, + Node: cfg.Node, ApiServer: config.ApiServer{ SubjectAltNames: cfg.SubjectAltNames, }, diff --git a/pkg/components/render.go b/pkg/components/render.go index 43c79e6478..d9d4f6d70c 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -23,8 +23,8 @@ var templateFuncs = map[string]interface{}{ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, - "NodeName": cfg.NodeName, - "NodeIP": cfg.NodeIP, + "NodeName": cfg.Node.HostnameOverride, + "NodeIP": cfg.Node.NodeIP, "ClusterCIDR": cfg.Cluster.ClusterCIDR, "ServiceCIDR": cfg.Cluster.ServiceCIDR, "ClusterDNS": cfg.Cluster.DNS, diff --git a/pkg/config/config.go b/pkg/config/config.go index 91a7895197..8dfd211194 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -89,8 +89,6 @@ type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into // the IP configured in the endpoint slice for kubernetes service. Must be @@ -105,7 +103,8 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` } // Top level config file @@ -245,8 +244,10 @@ func NewMicroshiftConfig() *MicroshiftConfig { return &MicroshiftConfig{ LogVLevel: 2, SubjectAltNames: subjectAltNames, - NodeName: strings.ToLower(nodeName), - NodeIP: nodeIP, + Node: Node{ + HostnameOverride: nodeName, + NodeIP: nodeIP, + }, DNS: DNS{ BaseDomain: "example.com", }, @@ -272,7 +273,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.NodeName == strings.ToLower(hostname) + return c.Node.HostnameOverride == hostname } // Read or set the NodeName that will be used for this MicroShift instance @@ -282,10 +283,10 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.NodeName, nil + return c.Node.HostnameOverride, nil } else if err != nil { return "", err } @@ -294,8 +295,8 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } establishedNodeName, err := c.establishNodeName() @@ -303,14 +304,14 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.NodeName { + if establishedNodeName != c.Node.HostnameOverride { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) + c.Node.HostnameOverride, establishedNodeName) } else { - c.NodeName = establishedNodeName + c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) } } @@ -395,12 +396,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = strings.ToLower(config.Node.HostnameOverride) - } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP - } + c.Node = config.Node if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -504,10 +500,10 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 804dcecdd7..711485f21a 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -66,10 +66,12 @@ func TestConfigFile(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", + LogVLevel: 4, + SubjectAltNames: []string{"node1", "node2"}, + Node: Node{ + HostnameOverride: "node1", + NodeIP: "1.2.3.4", + }, KASAdvertiseAddress: "6.7.8.9", DNS: DNS{ BaseDomain: "example.com", @@ -168,10 +170,12 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", + LogVLevel: 4, + SubjectAltNames: []string{"node1", "node2"}, + Node: Node{ + HostnameOverride: "node1", + NodeIP: "1.2.3.4", + }, KASAdvertiseAddress: "6.7.8.9", SkipKASInterface: true, DNS: DNS{ @@ -251,7 +255,7 @@ func TestMicroshiftConfigIsDefaultNodeName(t *testing.T) { t.Errorf("expected default IsDefaultNodeName to be true") } - c.NodeName += "-suffix" + c.Node.HostnameOverride += "-suffix" if c.isDefaultNodeName() { t.Errorf("expected default IsDefaultNodeName to be false") } @@ -262,7 +266,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { defer cleanup() c := NewMicroshiftConfig() - c.NodeName = "node1" + c.Node.HostnameOverride = "node1" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err != nil { t.Errorf("failed to validate node name on first call: %v", err) @@ -271,7 +275,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { nodeNameFile := filepath.Join(dataDir, ".nodename") if data, err := os.ReadFile(nodeNameFile); err != nil { t.Errorf("failed to read node name from file %q: %v", nodeNameFile, err) - } else if string(data) != c.NodeName { + } else if string(data) != c.Node.HostnameOverride { t.Errorf("node name file doesn't match the node name in the saved file: %v", err) } @@ -279,7 +283,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err == nil { t.Errorf("validation should have failed for nodename change: %v", err) } @@ -307,7 +311,7 @@ func TestMicroshiftConfigNodeNameValidationFromDefault(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err != nil { t.Errorf("validation should have failed in this case, it must be a warning in logs: %v", err) } @@ -318,7 +322,7 @@ func TestMicroshiftConfigNodeNameValidationBadName(t *testing.T) { defer cleanup() c := NewMicroshiftConfig() - c.NodeName = "1.2.3.4" + c.Node.HostnameOverride = "1.2.3.4" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err == nil { t.Errorf("failed to validate node name.") diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 56fd397051..2a5d9a0ad3 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -36,7 +36,7 @@ var _ servicemanager.Service = &LoadbalancerServiceController{} func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { return &LoadbalancerServiceController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), } } diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 7ff7e62b9b..3daee405c2 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -25,8 +25,8 @@ type MicroShiftmDNSController struct { func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { return &MicroShiftmDNSController{ - NodeIP: cfg.NodeIP, - NodeName: cfg.NodeName, + NodeIP: cfg.Node.NodeIP, + NodeName: cfg.Node.HostnameOverride, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), hostCount: make(map[string]int), } diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 31a75beb5f..c8e5c487a6 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -75,8 +75,8 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { kubeletFlags.BootstrapKubeconfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.KubeConfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.RuntimeCgroups = "/system.slice/crio.service" - kubeletFlags.HostnameOverride = cfg.NodeName - kubeletFlags.NodeIP = cfg.NodeIP + kubeletFlags.HostnameOverride = cfg.Node.HostnameOverride + kubeletFlags.NodeIP = cfg.Node.NodeIP kubeletFlags.ContainerRuntime = "remote" kubeletFlags.RemoteRuntimeEndpoint = "unix:///var/run/crio/crio.sock" kubeletFlags.NodeLabels["node-role.kubernetes.io/control-plane"] = "" diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index b68d159bd6..0c18891ff7 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -55,7 +55,7 @@ func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchContro } return &SysConfWatchController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, timerFd: fd, } } From e5d750d442f30d8985c81432757d5d08a557b36b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:56:12 -0500 Subject: [PATCH 03/10] USHIFT-936: align the Debugging portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 15 ++++++++------- pkg/cmd/showConfig.go | 9 +-------- pkg/config/config.go | 15 ++++++++------- pkg/config/config_test.go | 8 ++++++-- pkg/controllers/kube-apiserver.go | 2 +- pkg/controllers/kube-controller-manager.go | 2 +- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 8dfd211194..18b3009054 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -86,8 +86,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into @@ -103,8 +101,9 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` - Node Node `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` + Debugging Debugging `json:"debugging"` } // Top level config file @@ -242,7 +241,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { } return &MicroshiftConfig{ - LogVLevel: 2, + Debugging: Debugging{ + LogLevel: "Normal", + }, SubjectAltNames: subjectAltNames, Node: Node{ HostnameOverride: nodeName, @@ -395,8 +396,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { } // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() c.Node = config.Node + c.Debugging = config.Debugging if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -559,7 +560,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { +func (c *MicroshiftConfig) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index dd0eb19bbb..ed0bdc32c0 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -5,7 +5,6 @@ import ( "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" @@ -43,10 +42,6 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command } // map back from internal representation to user config - logLevels := []string{"", "", "Normal", "", "Debug", "", "Trace", "", "TraceAll"} - if cfg.LogVLevel < 0 || cfg.LogVLevel >= len(logLevels) { - klog.Fatal("logVLevel out of range [0..%d] %d", len(logLevels)-1, cfg.LogVLevel) - } userCfg := config.Config{ Network: config.Network{ ClusterNetwork: []config.ClusterNetworkEntry{ @@ -60,9 +55,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ApiServer: config.ApiServer{ SubjectAltNames: cfg.SubjectAltNames, }, - Debugging: config.Debugging{ - LogLevel: logLevels[cfg.LogVLevel], - }, + Debugging: cfg.Debugging, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 8dfd211194..18b3009054 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -86,8 +86,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into @@ -103,8 +101,9 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` - Node Node `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` + Debugging Debugging `json:"debugging"` } // Top level config file @@ -242,7 +241,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { } return &MicroshiftConfig{ - LogVLevel: 2, + Debugging: Debugging{ + LogLevel: "Normal", + }, SubjectAltNames: subjectAltNames, Node: Node{ HostnameOverride: nodeName, @@ -395,8 +396,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { } // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() c.Node = config.Node + c.Debugging = config.Debugging if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -559,7 +560,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { +func (c *MicroshiftConfig) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 711485f21a..42ae480122 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -66,7 +66,9 @@ func TestConfigFile(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, + Debugging: Debugging{ + LogLevel: "Debug", + }, SubjectAltNames: []string{"node1", "node2"}, Node: Node{ HostnameOverride: "node1", @@ -170,7 +172,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, + Debugging: Debugging{ + LogLevel: "Debug", + }, SubjectAltNames: []string{"node1", "node2"}, Node: Node{ HostnameOverride: "node1", diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index cdf6884c17..12babc785d 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -92,7 +92,7 @@ func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { - s.verbosity = cfg.LogVLevel + s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) kubeCSRSignerDir := cryptomaterial.CSRSignerCertDir(certsDir) diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 9c8a30b018..3c15528282 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -93,7 +93,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "use-service-account-credentials": {"true"}, "cluster-signing-cert-file": {clusterSigningCert}, "cluster-signing-key-file": {clusterSigningKey}, - "v": {strconv.Itoa(cfg.LogVLevel)}, + "v": {strconv.Itoa(cfg.GetVerbosity())}, }, } From 1c231316900a15d5091478ce36a7eb5e582bf107 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 17:11:24 -0500 Subject: [PATCH 04/10] USHIFT-936: align the Etcd portion of the config data structures --- etcd/cmd/microshift-etcd/run.go | 2 +- .../openshift/microshift/pkg/config/config.go | 63 +++++++++---------- pkg/cmd/showConfig.go | 1 + pkg/config/config.go | 63 +++++++++---------- pkg/config/config_test.go | 14 +++-- 5 files changed, 68 insertions(+), 75 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index c59a915ea1..d0f04bf33e 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -66,7 +66,7 @@ func (s *EtcdService) Name() string { return "etcd" } func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage - s.defragCheckFreq = cfg.Etcd.DefragCheckFreq + s.defragCheckFreq = cfg.Etcd.DefragCheckDuration s.doStartupDefrag = cfg.Etcd.DoStartupDefrag microshiftDataDir := config.GetDataDir() diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 18b3009054..945596b9b8 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -59,30 +59,23 @@ type IngressConfig struct { ServingKey []byte } -type InternalEtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - type EtcdConfig struct { // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string + QuotaBackendSize string `json:"quotaBackendSize"` + QuotaBackendBytes int64 `json:"-"` + // If the backend is fragmented more than `maxFragmentedPercentage` // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string - MaxFragmentedPercentage float64 + MinDefragSize string `json:"minDefragSize"` + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string + DefragCheckFreq string `json:"defragCheckFreq"` + DefragCheckDuration time.Duration `json:"-"` + // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool + DoStartupDefrag bool `json:"doStartupDefrag"` } type MicroshiftConfig struct { @@ -98,8 +91,8 @@ type MicroshiftConfig struct { SkipKASInterface bool `json:"-"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd InternalEtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd EtcdConfig `json:"etcd"` DNS DNS `json:"-"` Node Node `json:"-"` @@ -258,12 +251,15 @@ func NewMicroshiftConfig() *MicroshiftConfig { ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: InternalEtcdConfig{ - MinDefragBytes: 100 * 1024 * 1024, // 100MB + Etcd: EtcdConfig{ + MinDefragSize: "100Mi", + MinDefragBytes: 100 * 1024 * 1024, // 100MiB MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GB + QuotaBackendSize: "2Gi", + QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GiB }, } } @@ -415,15 +411,16 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } - if config.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) + c.Etcd = config.Etcd + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) if err != nil { return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) } - c.Etcd.DefragCheckFreq = d + c.Etcd.DefragCheckDuration = d } - if config.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) if err != nil { return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) } @@ -431,11 +428,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.MinDefragBytes = q.Value() } } - if config.Etcd.MaxFragmentedPercentage > 0 { - c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage - } - if config.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) if err != nil { return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) } @@ -443,7 +437,6 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.QuotaBackendBytes = q.Value() } } - c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag return nil } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index ed0bdc32c0..b88d0b661c 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -56,6 +56,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command SubjectAltNames: cfg.SubjectAltNames, }, Debugging: cfg.Debugging, + Etcd: cfg.Etcd, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 18b3009054..945596b9b8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,30 +59,23 @@ type IngressConfig struct { ServingKey []byte } -type InternalEtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - type EtcdConfig struct { // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string + QuotaBackendSize string `json:"quotaBackendSize"` + QuotaBackendBytes int64 `json:"-"` + // If the backend is fragmented more than `maxFragmentedPercentage` // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string - MaxFragmentedPercentage float64 + MinDefragSize string `json:"minDefragSize"` + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string + DefragCheckFreq string `json:"defragCheckFreq"` + DefragCheckDuration time.Duration `json:"-"` + // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool + DoStartupDefrag bool `json:"doStartupDefrag"` } type MicroshiftConfig struct { @@ -98,8 +91,8 @@ type MicroshiftConfig struct { SkipKASInterface bool `json:"-"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd InternalEtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd EtcdConfig `json:"etcd"` DNS DNS `json:"-"` Node Node `json:"-"` @@ -258,12 +251,15 @@ func NewMicroshiftConfig() *MicroshiftConfig { ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: InternalEtcdConfig{ - MinDefragBytes: 100 * 1024 * 1024, // 100MB + Etcd: EtcdConfig{ + MinDefragSize: "100Mi", + MinDefragBytes: 100 * 1024 * 1024, // 100MiB MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GB + QuotaBackendSize: "2Gi", + QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GiB }, } } @@ -415,15 +411,16 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } - if config.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) + c.Etcd = config.Etcd + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) if err != nil { return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) } - c.Etcd.DefragCheckFreq = d + c.Etcd.DefragCheckDuration = d } - if config.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) if err != nil { return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) } @@ -431,11 +428,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.MinDefragBytes = q.Value() } } - if config.Etcd.MaxFragmentedPercentage > 0 { - c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage - } - if config.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) if err != nil { return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) } @@ -443,7 +437,6 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.QuotaBackendBytes = q.Value() } } - c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 42ae480122..12be8b7798 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -84,11 +84,14 @@ func TestConfigFile(t *testing.T) { ServiceCIDR: "40.30.20.10/16", ServiceNodePortRange: "1024-32767", }, - Etcd: InternalEtcdConfig{ + Etcd: EtcdConfig{ + QuotaBackendSize: "2Gi", QuotaBackendBytes: 2 * 1024 * 1024 * 1024, + MinDefragSize: "100Mi", MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, }, }, @@ -192,11 +195,14 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { ServiceNodePortRange: "1024-32767", DNS: "40.30.0.10", }, - Etcd: InternalEtcdConfig{ + Etcd: EtcdConfig{ + QuotaBackendSize: "2Gi", QuotaBackendBytes: 2 * 1024 * 1024 * 1024, + MinDefragSize: "100Mi", MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, }, }, From 317a34c4564edaafb2d24e1064861de156a5625a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 17:24:05 -0500 Subject: [PATCH 05/10] USHIFT-936: align the ApiServer portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 54 +++++++++---------- pkg/cmd/init.go | 6 +-- pkg/cmd/showConfig.go | 8 ++- pkg/config/config.go | 54 +++++++++---------- pkg/config/config_test.go | 14 +++-- pkg/controllers/kube-apiserver.go | 2 +- pkg/node/netconfig.go | 4 +- 7 files changed, 66 insertions(+), 76 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 945596b9b8..c525d35eae 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -79,17 +79,7 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - Cluster ClusterConfig `json:"cluster"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -97,6 +87,7 @@ type MicroshiftConfig struct { DNS DNS `json:"-"` Node Node `json:"-"` Debugging Debugging `json:"debugging"` + ApiServer ApiServer `json:"-"` } // Top level config file @@ -150,9 +141,15 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` } type Node struct { @@ -237,7 +234,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { Debugging: Debugging{ LogLevel: "Normal", }, - SubjectAltNames: subjectAltNames, + ApiServer: ApiServer{ + SubjectAltNames: subjectAltNames, + }, Node: Node{ HostnameOverride: nodeName, NodeIP: nodeIP, @@ -404,12 +403,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } c.DNS = config.DNS - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames - } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress - } + c.ApiServer = config.ApiServer c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -459,20 +453,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // If KAS advertise address is not configured then grab it from the service // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) } - c.KASAdvertiseAddress = apiServerServiceIP.String() - c.SkipKASInterface = false + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false } else { - c.SkipKASInterface = true + c.ApiServer.SkipInterface = true } - if len(c.SubjectAltNames) > 0 { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -490,20 +484,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -512,7 +506,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 92ad9e3694..79faf8bb9f 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -74,14 +74,14 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err cfg.Node.HostnameOverride, "api." + cfg.DNS.BaseDomain, } - externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) + externalCertNames = append(externalCertNames, cfg.ApiServer.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add // it to the certificates or else the internal pod access to apiserver is // broken. Because of client-go not using SNI and the way apiserver handles // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.Node.NodeIP { + if cfg.ApiServer.AdvertiseAddress != cfg.Node.NodeIP { externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } @@ -390,7 +390,7 @@ func initKubeconfigs( } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.Node.HostnameOverride, "localhost") { + for _, name := range append(cfg.ApiServer.SubjectAltNames, cfg.Node.HostnameOverride, "localhost") { u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index b88d0b661c..95848e3de3 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -50,11 +50,9 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: cfg.DNS, - Node: cfg.Node, - ApiServer: config.ApiServer{ - SubjectAltNames: cfg.SubjectAltNames, - }, + DNS: cfg.DNS, + Node: cfg.Node, + ApiServer: cfg.ApiServer, Debugging: cfg.Debugging, Etcd: cfg.Etcd, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 945596b9b8..c525d35eae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -79,17 +79,7 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - Cluster ClusterConfig `json:"cluster"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -97,6 +87,7 @@ type MicroshiftConfig struct { DNS DNS `json:"-"` Node Node `json:"-"` Debugging Debugging `json:"debugging"` + ApiServer ApiServer `json:"-"` } // Top level config file @@ -150,9 +141,15 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` } type Node struct { @@ -237,7 +234,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { Debugging: Debugging{ LogLevel: "Normal", }, - SubjectAltNames: subjectAltNames, + ApiServer: ApiServer{ + SubjectAltNames: subjectAltNames, + }, Node: Node{ HostnameOverride: nodeName, NodeIP: nodeIP, @@ -404,12 +403,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } c.DNS = config.DNS - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames - } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress - } + c.ApiServer = config.ApiServer c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -459,20 +453,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // If KAS advertise address is not configured then grab it from the service // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) } - c.KASAdvertiseAddress = apiServerServiceIP.String() - c.SkipKASInterface = false + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false } else { - c.SkipKASInterface = true + c.ApiServer.SkipInterface = true } - if len(c.SubjectAltNames) > 0 { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -490,20 +484,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -512,7 +506,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 12be8b7798..0e6cf09314 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -69,12 +69,14 @@ func TestConfigFile(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - SubjectAltNames: []string{"node1", "node2"}, + ApiServer: ApiServer{ + SubjectAltNames: []string{"node1", "node2"}, + AdvertiseAddress: "6.7.8.9", + }, Node: Node{ HostnameOverride: "node1", NodeIP: "1.2.3.4", }, - KASAdvertiseAddress: "6.7.8.9", DNS: DNS{ BaseDomain: "example.com", }, @@ -178,13 +180,15 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - SubjectAltNames: []string{"node1", "node2"}, + ApiServer: ApiServer{ + SubjectAltNames: []string{"node1", "node2"}, + AdvertiseAddress: "6.7.8.9", + SkipInterface: true, + }, Node: Node{ HostnameOverride: "node1", NodeIP: "1.2.3.4", }, - KASAdvertiseAddress: "6.7.8.9", - SkipKASInterface: true, DNS: DNS{ BaseDomain: "example.com", }, diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 12babc785d..c627c5f7ce 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -117,7 +117,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { s.masterURL = cfg.Cluster.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) - s.advertiseAddress = cfg.KASAdvertiseAddress + s.advertiseAddress = cfg.ApiServer.AdvertiseAddress overrides := &kubecontrolplanev1.KubeAPIServerConfig{ APIServerArguments: map[string]kubecontrolplanev1.Arguments{ diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e43813d7f8..e92f9c81d3 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -48,8 +48,8 @@ func (n *NetworkConfiguration) Name() string { return componentNetwork func (n *NetworkConfiguration) Dependencies() []string { return []string{} } func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { - n.kasAdvertiseAddress = cfg.KASAdvertiseAddress - n.skipInterfaceConfiguration = cfg.SkipKASInterface + n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress + n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { From dcef4c2d03424e02328764189493ec73437b72d9 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 18:23:53 -0500 Subject: [PATCH 06/10] USHIFT-936: align the Cluster/Network portions of the config data structures Also introduces assert package in the test code for easier diagnosis of unit test failures. --- .../openshift/microshift/pkg/config/config.go | 67 ++++++++++------- pkg/cmd/init.go | 18 ++--- pkg/cmd/run.go | 4 +- pkg/cmd/showConfig.go | 6 +- pkg/components/controllers.go | 2 +- pkg/components/render.go | 6 +- pkg/config/config.go | 73 +++++++++++-------- pkg/config/config_test.go | 36 +++++---- pkg/controllers/kube-apiserver.go | 10 +-- pkg/controllers/kube-controller-manager.go | 2 +- pkg/node/kubelet.go | 2 +- 11 files changed, 132 insertions(+), 94 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index c525d35eae..5d1a2c7a05 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -47,11 +47,9 @@ var ( ) type ClusterConfig struct { - URL string `json:"-"` ClusterCIDR string `json:"clusterCIDR"` ServiceCIDR string `json:"serviceCIDR"` ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` } type IngressConfig struct { @@ -79,8 +77,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -88,6 +84,7 @@ type MicroshiftConfig struct { Node Node `json:"-"` Debugging Debugging `json:"debugging"` ApiServer ApiServer `json:"-"` + Network Network `json:"-"` } // Top level config file @@ -118,6 +115,9 @@ type Network struct { // installed. // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` + + // The DNS server to use + DNS string `json:"-"` } type ClusterNetworkEntry struct { @@ -150,6 +150,9 @@ type ApiServer struct { // Determines if kube-apiserver controller should configure the // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` } type Node struct { @@ -236,6 +239,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { }, ApiServer: ApiServer{ SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: nodeName, @@ -244,11 +248,17 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", }, Etcd: EtcdConfig{ MinDefragSize: "100Mi", @@ -315,10 +325,10 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { +func (c *MicroshiftConfig) ApiServerPort() (int, error) { var port string - parsed, err := url.Parse(c.URL) + parsed, err := url.Parse(c.ApiServer.URL) if err != nil { return 0, err } @@ -393,17 +403,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.Node = config.Node c.Debugging = config.Debugging - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } + c.DNS = config.DNS c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -435,6 +442,19 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } +func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + // Note: add a configFile parameter here because of unit test requiring custom // local directory func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { @@ -444,18 +464,15 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } } - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } - c.Cluster.DNS = clusterDNS // If KAS advertise address is not configured then grab it from the service // CIDR automatically. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) @@ -479,7 +496,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 79faf8bb9f..d5faea3f98 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -60,7 +60,7 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { - _, svcNet, err := net.ParseCIDR(cfg.Cluster.ServiceCIDR) + _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err } @@ -380,11 +380,11 @@ func initKubeconfigs( return err } - u, err := url.Parse(cfg.Cluster.URL) + u, err := url.Parse(cfg.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } - apiServerPort, err := cfg.Cluster.ApiServerPort() + apiServerPort, err := cfg.ApiServerPort() if err != nil { return fmt.Errorf("failed to get apiserver port: %v", err) } @@ -405,7 +405,7 @@ func initKubeconfigs( if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeAdmin), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, adminKubeconfigCertPEM, adminKubeconfigKeyPEM, @@ -419,7 +419,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, kcmCertPEM, kcmKeyPEM, @@ -433,7 +433,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeScheduler), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, schedulerCertPEM, schedulerKeyPEM, ); err != nil { @@ -446,7 +446,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.Kubelet), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, kubeletCertPEM, kubeletKeyPEM, ); err != nil { @@ -458,7 +458,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.ClusterPolicyController), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, clusterPolicyControllerCertPEM, clusterPolicyControllerKeyPEM, ); err != nil { @@ -471,7 +471,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.RouteControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, routeControllerManagerCertPEM, routeControllerManagerKeyPEM, ); err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 0a5212c1ea..cf9294a644 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -58,8 +58,8 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { if err := util.AddToNoProxyEnv( cfg.Node.NodeIP, cfg.Node.HostnameOverride, - cfg.Cluster.ClusterCIDR, - cfg.Cluster.ServiceCIDR, + cfg.Network.ClusterNetwork[0].CIDR, + cfg.Network.ServiceNetwork[0], ".svc", ".cluster.local", "."+cfg.DNS.BaseDomain); err != nil { diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 95848e3de3..1ec4bbbd96 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -45,10 +45,10 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command userCfg := config.Config{ Network: config.Network{ ClusterNetwork: []config.ClusterNetworkEntry{ - {CIDR: cfg.Cluster.ClusterCIDR}, + {CIDR: cfg.Network.ClusterNetwork[0].CIDR}, }, - ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, - ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServiceNetwork: []string{cfg.Network.ServiceNetwork[0]}, + ServiceNodePortRange: cfg.Network.ServiceNodePortRange, }, DNS: cfg.DNS, Node: cfg.Node, diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index 10ae6593c1..b69f965cf7 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -210,7 +210,7 @@ func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) err } extraParams := assets.RenderParams{ - "ClusterIP": cfg.Cluster.DNS, + "ClusterIP": cfg.Network.DNS, } if err := assets.ApplyServices(svc, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply service %v %v", svc, err) diff --git a/pkg/components/render.go b/pkg/components/render.go index d9d4f6d70c..b545d920de 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -25,9 +25,9 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar "ReleaseImage": release.Image, "NodeName": cfg.Node.HostnameOverride, "NodeIP": cfg.Node.NodeIP, - "ClusterCIDR": cfg.Cluster.ClusterCIDR, - "ServiceCIDR": cfg.Cluster.ServiceCIDR, - "ClusterDNS": cfg.Cluster.DNS, + "ClusterCIDR": cfg.Network.ClusterNetwork[0].CIDR, + "ServiceCIDR": cfg.Network.ServiceNetwork[0], + "ClusterDNS": cfg.Network.DNS, "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { diff --git a/pkg/config/config.go b/pkg/config/config.go index c525d35eae..89bc81cb58 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -46,14 +46,6 @@ var ( manifestsDir = findManifestsDir() ) -type ClusterConfig struct { - URL string `json:"-"` - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` -} - type IngressConfig struct { ServingCertificate []byte ServingKey []byte @@ -79,8 +71,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -88,6 +78,7 @@ type MicroshiftConfig struct { Node Node `json:"-"` Debugging Debugging `json:"debugging"` ApiServer ApiServer `json:"-"` + Network Network `json:"-"` } // Top level config file @@ -118,6 +109,9 @@ type Network struct { // installed. // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` + + // The DNS server to use + DNS string `json:"-"` } type ClusterNetworkEntry struct { @@ -150,6 +144,9 @@ type ApiServer struct { // Determines if kube-apiserver controller should configure the // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` } type Node struct { @@ -236,6 +233,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { }, ApiServer: ApiServer{ SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: nodeName, @@ -244,11 +242,17 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", }, Etcd: EtcdConfig{ MinDefragSize: "100Mi", @@ -315,10 +319,10 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { +func (c *MicroshiftConfig) ApiServerPort() (int, error) { var port string - parsed, err := url.Parse(c.URL) + parsed, err := url.Parse(c.ApiServer.URL) if err != nil { return 0, err } @@ -393,17 +397,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.Node = config.Node c.Debugging = config.Debugging - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } + c.DNS = config.DNS c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -435,6 +436,19 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } +func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + // Note: add a configFile parameter here because of unit test requiring custom // local directory func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { @@ -444,18 +458,15 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } } - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } - c.Cluster.DNS = clusterDNS // If KAS advertise address is not configured then grab it from the service // CIDR automatically. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) @@ -479,7 +490,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0e6cf09314..999ad3bc4b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3,11 +3,12 @@ package config import ( "os" "path/filepath" - "reflect" "testing" "time" "sigs.k8s.io/yaml" + + "github.com/stretchr/testify/assert" ) const ( @@ -72,6 +73,7 @@ func TestConfigFile(t *testing.T) { ApiServer: ApiServer{ SubjectAltNames: []string{"node1", "node2"}, AdvertiseAddress: "6.7.8.9", + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: "node1", @@ -80,11 +82,15 @@ func TestConfigFile(t *testing.T) { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", + }, + }, + ServiceNetwork: []string{"40.30.20.10/16"}, ServiceNodePortRange: "1024-32767", + DNS: "40.30.0.10", }, Etcd: EtcdConfig{ QuotaBackendSize: "2Gi", @@ -123,8 +129,8 @@ func TestConfigFile(t *testing.T) { if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadFromConfigFile() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + assert.Equal(t, tt.expected, *config) } }) } @@ -184,6 +190,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"node1", "node2"}, AdvertiseAddress: "6.7.8.9", SkipInterface: true, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: "node1", @@ -192,10 +199,13 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", + }, + }, + ServiceNetwork: []string{"40.30.20.10/16"}, ServiceNodePortRange: "1024-32767", DNS: "40.30.0.10", }, @@ -256,8 +266,8 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadAndValidate() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + assert.Equal(t, tt.expected, *config) } }) } diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index c627c5f7ce..16e9db4f61 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -110,12 +110,12 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { } // Get the apiserver port so we can set it as an argument - apiServerPort, err := cfg.Cluster.ApiServerPort() + apiServerPort, err := cfg.ApiServerPort() if err != nil { return err } - s.masterURL = cfg.Cluster.URL + s.masterURL = cfg.ApiServer.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) s.advertiseAddress = cfg.ApiServer.AdvertiseAddress @@ -139,7 +139,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { "proxy-client-key-file": {cryptomaterial.ClientKeyPath(aggregatorClientCertDir)}, "requestheader-client-ca-file": {aggregatorCAPath}, "service-account-signing-key-file": {microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key"}, - "service-node-port-range": {cfg.Cluster.ServiceNodePortRange}, + "service-node-port-range": {cfg.Network.ServiceNodePortRange}, "tls-cert-file": {servingCert}, "tls-private-key-file": {servingKey}, "disable-admission-plugins": { @@ -214,8 +214,8 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { ServiceAccountPublicKeyFiles: []string{ microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.pub", }, - ServicesSubnet: cfg.Cluster.ServiceCIDR, - ServicesNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServicesSubnet: cfg.Network.ServiceNetwork[0], + ServicesNodePortRange: cfg.Network.ServiceNodePortRange, } overridesBytes, err := json.Marshal(overrides) diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 3c15528282..70ebdff2ac 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -85,7 +85,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "authorization-kubeconfig": {kubeConfig}, "service-account-private-key-file": {kcmServiceAccountPrivateKeyFile()}, "allocate-node-cidrs": {"true"}, - "cluster-cidr": {cfg.Cluster.ClusterCIDR}, + "cluster-cidr": {cfg.Network.ClusterNetwork[0].CIDR}, "root-ca-file": {kcmRootCAFile()}, "bind-address": {"127.0.0.1"}, "secure-port": {"10257"}, diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index c8e5c487a6..ebc246ded8 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -112,7 +112,7 @@ cgroupDriver: "systemd" failSwapOn: false volumePluginDir: ` + microshiftDataDir + `/kubelet-plugins/volume/exec clusterDNS: - - ` + cfg.Cluster.DNS + ` + - ` + cfg.Network.DNS + ` clusterDomain: cluster.local containerLogMaxSize: 50Mi maxPods: 250 From d43a5050d63f697cd08b56d858f60256f77586ca Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 18:36:13 -0500 Subject: [PATCH 07/10] USHIFT-936: merge MicroshiftConfig into Config --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 43 ++++++++----------- pkg/assets/crd.go | 4 +- pkg/cmd/init.go | 6 +-- pkg/cmd/run.go | 2 +- pkg/components/components.go | 2 +- pkg/components/controllers.go | 6 +-- pkg/components/networking.go | 2 +- pkg/components/render.go | 2 +- pkg/components/storage.go | 2 +- pkg/config/config.go | 43 ++++++++----------- pkg/config/config_test.go | 12 +++--- pkg/controllers/cluster-policy-controller.go | 4 +- pkg/controllers/etcd.go | 2 +- pkg/controllers/infra-services-controller.go | 6 +-- pkg/controllers/kube-apiserver.go | 6 +-- pkg/controllers/kube-controller-manager.go | 4 +- pkg/controllers/kube-scheduler.go | 6 +-- pkg/controllers/openshift-crd-manager.go | 4 +- .../openshift-default-scc-manager.go | 6 +-- .../openshift-route-controller-manager.go | 6 +-- pkg/controllers/version.go | 4 +- pkg/kustomize/apply.go | 2 +- pkg/loadbalancerservice/controller.go | 2 +- pkg/mdns/controller.go | 2 +- pkg/node/kubelet.go | 6 +-- pkg/node/netconfig.go | 4 +- pkg/sysconfwatch/sysconfwatch_linux.go | 2 +- 28 files changed, 88 insertions(+), 106 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index d0f04bf33e..67f3cb9a41 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -55,7 +55,7 @@ type EtcdService struct { doStartupDefrag bool } -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { s := &EtcdService{} s.configure(cfg) return s @@ -63,7 +63,7 @@ func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { func (s *EtcdService) Name() string { return "etcd" } -func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { +func (s *EtcdService) configure(cfg *config.Config) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage s.defragCheckFreq = cfg.Etcd.DefragCheckDuration diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 5d1a2c7a05..bc8437a1e9 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -76,25 +76,16 @@ type EtcdConfig struct { DoStartupDefrag bool `json:"doStartupDefrag"` } -type MicroshiftConfig struct { - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` - - DNS DNS `json:"-"` - Node Node `json:"-"` - Debugging Debugging `json:"debugging"` - ApiServer ApiServer `json:"-"` - Network Network `json:"-"` -} - -// Top level config file type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` Node Node `json:"node"` ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` + + // Internal-only fields + Ingress IngressConfig `json:"-"` } type Network struct { @@ -195,11 +186,11 @@ const ( ) // KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { return filepath.Join(dataDir, "resources", string(id), "kubeconfig") } -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { +func (cfg *Config) KubeConfigAdminPath(id string) string { return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") } @@ -219,7 +210,7 @@ func getAllHostnames() ([]string, error) { return set.List(), nil } -func NewMicroshiftConfig() *MicroshiftConfig { +func NewMicroshiftConfig() *Config { nodeName, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -233,7 +224,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { klog.Fatalf("failed to get all hostnames: %v", err) } - return &MicroshiftConfig{ + return &Config{ Debugging: Debugging{ LogLevel: "Normal", }, @@ -274,7 +265,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { } // Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { +func (c *Config) isDefaultNodeName() bool { hostname, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -283,7 +274,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { } // Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { +func (c *Config) establishNodeName() (string, error) { filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { @@ -300,7 +291,7 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { } // Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { +func (c *Config) validateNodeName(isDefaultNodeName bool) error { if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } @@ -325,7 +316,7 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *MicroshiftConfig) ApiServerPort() (int, error) { +func (c *Config) ApiServerPort() (int, error) { var port string parsed, err := url.Parse(c.ApiServer.URL) @@ -390,7 +381,7 @@ func StringInList(s string, list []string) bool { return false } -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { +func (c *Config) ReadFromConfigFile(configFile string) error { contents, err := os.ReadFile(configFile) if err != nil { return fmt.Errorf("reading config file %q: %v", configFile, err) @@ -400,7 +391,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return fmt.Errorf("decoding config file %s: %v", configFile, err) } - // Wire new Config type to existing MicroshiftConfig + // Wire new Config type to existing Config c.Node = config.Node c.Debugging = config.Debugging c.Network = config.Network @@ -442,7 +433,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } -func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { +func (c *Config) computeAndUpdateClusterDNS() error { if len(c.Network.ServiceNetwork) == 0 { return fmt.Errorf("network.serviceNetwork not filled in") } @@ -457,7 +448,7 @@ func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { // Note: add a configFile parameter here because of unit test requiring custom // local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { +func (c *Config) ReadAndValidate(configFile string) error { if configFile != "" { if err := c.ReadFromConfigFile(configFile); err != nil { return err @@ -564,7 +555,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *MicroshiftConfig) GetVerbosity() int { +func (c *Config) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/assets/crd.go b/pkg/assets/crd.go index 392e2b9940..077f5f5507 100644 --- a/pkg/assets/crd.go +++ b/pkg/assets/crd.go @@ -70,7 +70,7 @@ func isEstablished(cs *apiextclientv1.ApiextensionsV1Client, obj apiruntime.Obje return false, err } -func WaitForCrdsEstablished(cfg *config.MicroshiftConfig) error { +func WaitForCrdsEstablished(cfg *config.Config) error { restConfig, err := clientcmd.BuildConfigFromFlags("", cfg.KubeConfigPath(config.KubeAdmin)) if err != nil { return err @@ -115,7 +115,7 @@ func applyCRD(client *apiextclientv1.ApiextensionsV1Client, crd *apiextv1.Custom return err } -func ApplyCRDs(cfg *config.MicroshiftConfig) error { +func ApplyCRDs(cfg *config.Config) error { lock.Lock() defer lock.Unlock() diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index d5faea3f98..745c835a34 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -36,7 +36,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func initCerts(cfg *config.Config) (*certchains.CertificateChains, error) { certChains, err := certSetup(cfg) if err != nil { return nil, err @@ -59,7 +59,7 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err return certChains, err } -func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func certSetup(cfg *config.Config) (*certchains.CertificateChains, error) { _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err @@ -367,7 +367,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func initKubeconfigs( - cfg *config.MicroshiftConfig, + cfg *config.Config, certChains *certchains.CertificateChains, ) error { inClusterTrustBundlePEM, err := os.ReadFile(cryptomaterial.ServiceAccountTokenCABundlePath(cryptomaterial.CertsDirectory(microshiftDataDir))) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index cf9294a644..b8d72e3405 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -41,7 +41,7 @@ func NewRunMicroshiftCommand() *cobra.Command { return cmd } -func RunMicroshift(cfg *config.MicroshiftConfig) error { +func RunMicroshift(cfg *config.Config) error { if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { klog.Fatalf("Error in reading or validating configuration: %v", err) } diff --git a/pkg/components/components.go b/pkg/components/components.go index c20b5f2fc9..aaadbca36f 100755 --- a/pkg/components/components.go +++ b/pkg/components/components.go @@ -7,7 +7,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func StartComponents(cfg *config.MicroshiftConfig) error { +func StartComponents(cfg *config.Config) error { kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin) if err := startServiceCAController(cfg, kubeAdminConfig); err != nil { diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index b69f965cf7..13c4038980 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -9,7 +9,7 @@ import ( "k8s.io/klog/v2" ) -func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startServiceCAController(cfg *config.Config, kubeconfigPath string) error { var ( //TODO: fix the rolebinding and sa clusterRoleBinding = []string{ @@ -101,7 +101,7 @@ func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath strin return nil } -func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startIngressController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-router/cluster-role-binding.yaml", @@ -178,7 +178,7 @@ func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) return nil } -func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startDNSController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-dns/dns/cluster-role-binding.yaml", diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 0cefcd2e46..d0c75b2435 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -10,7 +10,7 @@ import ( "k8s.io/klog/v2" ) -func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCNIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/ovn/namespace.yaml", diff --git a/pkg/components/render.go b/pkg/components/render.go index b545d920de..5b466cc272 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -20,7 +20,7 @@ var templateFuncs = map[string]interface{}{ "Sha256sum": func(s string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) }, } -func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { +func renderParamsFromConfig(cfg *config.Config, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, "NodeName": cfg.Node.HostnameOverride, diff --git a/pkg/components/storage.go b/pkg/components/storage.go index f7e55367d9..75a2e53821 100644 --- a/pkg/components/storage.go +++ b/pkg/components/storage.go @@ -24,7 +24,7 @@ func getCSIPluginConfig() (*lvmd.Lvmd, error) { return (&lvmd.Lvmd{}).WithDefaults(), nil } -func startCSIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCSIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/lvms/topolvm-openshift-storage_namespace.yaml", diff --git a/pkg/config/config.go b/pkg/config/config.go index 89bc81cb58..5f969305c1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -70,25 +70,16 @@ type EtcdConfig struct { DoStartupDefrag bool `json:"doStartupDefrag"` } -type MicroshiftConfig struct { - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` - - DNS DNS `json:"-"` - Node Node `json:"-"` - Debugging Debugging `json:"debugging"` - ApiServer ApiServer `json:"-"` - Network Network `json:"-"` -} - -// Top level config file type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` Node Node `json:"node"` ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` + + // Internal-only fields + Ingress IngressConfig `json:"-"` } type Network struct { @@ -189,11 +180,11 @@ const ( ) // KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { return filepath.Join(dataDir, "resources", string(id), "kubeconfig") } -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { +func (cfg *Config) KubeConfigAdminPath(id string) string { return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") } @@ -213,7 +204,7 @@ func getAllHostnames() ([]string, error) { return set.List(), nil } -func NewMicroshiftConfig() *MicroshiftConfig { +func NewMicroshiftConfig() *Config { nodeName, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -227,7 +218,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { klog.Fatalf("failed to get all hostnames: %v", err) } - return &MicroshiftConfig{ + return &Config{ Debugging: Debugging{ LogLevel: "Normal", }, @@ -268,7 +259,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { } // Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { +func (c *Config) isDefaultNodeName() bool { hostname, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -277,7 +268,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { } // Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { +func (c *Config) establishNodeName() (string, error) { filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { @@ -294,7 +285,7 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { } // Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { +func (c *Config) validateNodeName(isDefaultNodeName bool) error { if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } @@ -319,7 +310,7 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *MicroshiftConfig) ApiServerPort() (int, error) { +func (c *Config) ApiServerPort() (int, error) { var port string parsed, err := url.Parse(c.ApiServer.URL) @@ -384,7 +375,7 @@ func StringInList(s string, list []string) bool { return false } -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { +func (c *Config) ReadFromConfigFile(configFile string) error { contents, err := os.ReadFile(configFile) if err != nil { return fmt.Errorf("reading config file %q: %v", configFile, err) @@ -394,7 +385,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return fmt.Errorf("decoding config file %s: %v", configFile, err) } - // Wire new Config type to existing MicroshiftConfig + // Wire new Config type to existing Config c.Node = config.Node c.Debugging = config.Debugging c.Network = config.Network @@ -436,7 +427,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } -func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { +func (c *Config) computeAndUpdateClusterDNS() error { if len(c.Network.ServiceNetwork) == 0 { return fmt.Errorf("network.serviceNetwork not filled in") } @@ -451,7 +442,7 @@ func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { // Note: add a configFile parameter here because of unit test requiring custom // local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { +func (c *Config) ReadAndValidate(configFile string) error { if configFile != "" { if err := c.ReadFromConfigFile(configFile); err != nil { return err @@ -558,7 +549,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *MicroshiftConfig) GetVerbosity() int { +func (c *Config) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 999ad3bc4b..a63df0a095 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,7 +30,7 @@ func setupSuiteDataDir(t *testing.T) func() { func TestConfigFile(t *testing.T) { var ttests = []struct { config Config - expected MicroshiftConfig + expected Config expectErr bool }{ { @@ -66,7 +66,7 @@ func TestConfigFile(t *testing.T) { DoStartupDefrag: true, }, }, - expected: MicroshiftConfig{ + expected: Config{ Debugging: Debugging{ LogLevel: "Debug", }, @@ -145,7 +145,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { var ttests = []struct { name string config Config - expected MicroshiftConfig + expected Config expectErr bool }{ { @@ -182,7 +182,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { DoStartupDefrag: true, }, }, - expected: MicroshiftConfig{ + expected: Config{ Debugging: Debugging{ LogLevel: "Debug", }, @@ -229,7 +229,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"127.0.0.1", "localhost"}, }, }, - expected: MicroshiftConfig{}, + expected: Config{}, expectErr: true, }, { @@ -239,7 +239,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"kubernetes"}, }, }, - expected: MicroshiftConfig{}, + expected: Config{}, expectErr: true, }, } diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index d94fc54de0..45ae5e9f7f 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -35,7 +35,7 @@ type ClusterPolicyController struct { configErr error } -func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyController { +func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { s := &ClusterPolicyController{} s.configErr = s.configure(cfg) return s @@ -44,7 +44,7 @@ func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyCont func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *ClusterPolicyController) configure(cfg *config.MicroshiftConfig) error { +func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) scheme := runtime.NewScheme() diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index 5596e8cb25..d68b99219f 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -38,7 +38,7 @@ var ( type EtcdService struct{} -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { return &EtcdService{} } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 08a5c85298..7c5f8af766 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -26,10 +26,10 @@ import ( ) type InfrastructureServicesManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewInfrastructureServices(cfg *config.MicroshiftConfig) *InfrastructureServicesManager { +func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManager { s := &InfrastructureServicesManager{} s.cfg = cfg return s @@ -62,7 +62,7 @@ func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- st return ctx.Err() } -func applyDefaultRBACs(cfg *config.MicroshiftConfig) error { +func applyDefaultRBACs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( cr = []string{ diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 16e9db4f61..e46bab8d3f 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -80,7 +80,7 @@ type KubeAPIServer struct { advertiseAddress string } -func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { +func NewKubeAPIServer(cfg *config.Config) *KubeAPIServer { s := &KubeAPIServer{} if err := s.configure(cfg); err != nil { s.configureErr = err @@ -91,7 +91,7 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } -func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configure(cfg *config.Config) error { s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -255,7 +255,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { return nil } -func (s *KubeAPIServer) configureAuditPolicy(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configureAuditPolicy(cfg *config.Config) error { data := []byte(` apiVersion: audit.k8s.io/v1 kind: Policy diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 70ebdff2ac..c256291fb0 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -49,7 +49,7 @@ type KubeControllerManager struct { configureErr error } -func NewKubeControllerManager(cfg *config.MicroshiftConfig) *KubeControllerManager { +func NewKubeControllerManager(cfg *config.Config) *KubeControllerManager { s := &KubeControllerManager{} // TODO: manage and invoke the configure bits independently outside of this. s.args, s.applyFn, s.configureErr = configure(cfg) @@ -74,7 +74,7 @@ func kcmServiceAccountPrivateKeyFile() string { return microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key" } -func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() error, err error) { +func configure(cfg *config.Config) (args []string, applyFn func() error, err error) { kubeConfig := cfg.KubeConfigPath(config.KubeControllerManager) clusterSigningKey, clusterSigningCert := kcmClusterSigningCertKeyAndFile() diff --git a/pkg/controllers/kube-scheduler.go b/pkg/controllers/kube-scheduler.go index 5b0889fa54..7c4ff950a8 100644 --- a/pkg/controllers/kube-scheduler.go +++ b/pkg/controllers/kube-scheduler.go @@ -40,7 +40,7 @@ type KubeScheduler struct { kubeconfig string } -func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { +func NewKubeScheduler(cfg *config.Config) *KubeScheduler { s := &KubeScheduler{} s.configure(cfg) return s @@ -49,7 +49,7 @@ func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { func (s *KubeScheduler) Name() string { return "kube-scheduler" } func (s *KubeScheduler) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { +func (s *KubeScheduler) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("failed to write kube-scheduler config: %v", err) } @@ -61,7 +61,7 @@ func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { s.kubeconfig = cfg.KubeConfigPath(config.KubeScheduler) } -func (s *KubeScheduler) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeScheduler) writeConfig(cfg *config.Config) error { data := []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: diff --git a/pkg/controllers/openshift-crd-manager.go b/pkg/controllers/openshift-crd-manager.go index f505b650d0..de328afc89 100644 --- a/pkg/controllers/openshift-crd-manager.go +++ b/pkg/controllers/openshift-crd-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftCRDManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftCRDManager(cfg *config.MicroshiftConfig) *OpenShiftCRDManager { +func NewOpenShiftCRDManager(cfg *config.Config) *OpenShiftCRDManager { s := &OpenShiftCRDManager{} s.cfg = cfg return s diff --git a/pkg/controllers/openshift-default-scc-manager.go b/pkg/controllers/openshift-default-scc-manager.go index d933f2015a..19615724c1 100644 --- a/pkg/controllers/openshift-default-scc-manager.go +++ b/pkg/controllers/openshift-default-scc-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftDefaultSCCManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftDefaultSCCManager(cfg *config.MicroshiftConfig) *OpenShiftDefaultSCCManager { +func NewOpenShiftDefaultSCCManager(cfg *config.Config) *OpenShiftDefaultSCCManager { s := &OpenShiftDefaultSCCManager{} s.cfg = cfg return s @@ -51,7 +51,7 @@ func (s *OpenShiftDefaultSCCManager) Run(ctx context.Context, ready chan<- struc return ctx.Err() } -func ApplyDefaultSCCs(cfg *config.MicroshiftConfig) error { +func ApplyDefaultSCCs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( clusterRole = []string{ diff --git a/pkg/controllers/openshift-route-controller-manager.go b/pkg/controllers/openshift-route-controller-manager.go index bf7df109f0..3bfa301fab 100644 --- a/pkg/controllers/openshift-route-controller-manager.go +++ b/pkg/controllers/openshift-route-controller-manager.go @@ -44,7 +44,7 @@ const ( componentRCM = "route-controller-manager" ) -func NewRouteControllerManager(cfg *config.MicroshiftConfig) *OCPRouteControllerManager { +func NewRouteControllerManager(cfg *config.Config) *OCPRouteControllerManager { s := &OCPRouteControllerManager{} s.configure(cfg) return s @@ -55,13 +55,13 @@ func (s *OCPRouteControllerManager) Dependencies() []string { return []string{"kube-apiserver", "openshift-crd-manager"} } -func (s *OCPRouteControllerManager) configure(cfg *config.MicroshiftConfig) { +func (s *OCPRouteControllerManager) configure(cfg *config.Config) { s.kubeconfig = cfg.KubeConfigPath(config.RouteControllerManager) s.kubeadmconfig = cfg.KubeConfigPath(config.KubeAdmin) s.config = s.writeConfig(cfg) } -func (s *OCPRouteControllerManager) writeConfig(cfg *config.MicroshiftConfig) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { +func (s *OCPRouteControllerManager) writeConfig(cfg *config.Config) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { servingCertDir := cryptomaterial.RouteControllerManagerServingCertDir(cryptomaterial.CertsDirectory(microshiftDataDir)) c := &openshiftcontrolplanev1.OpenShiftControllerManagerConfig{ diff --git a/pkg/controllers/version.go b/pkg/controllers/version.go index adb7e04c58..4495cbcc8f 100644 --- a/pkg/controllers/version.go +++ b/pkg/controllers/version.go @@ -25,10 +25,10 @@ import ( ) type VersionManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewVersionManager(cfg *config.MicroshiftConfig) *VersionManager { +func NewVersionManager(cfg *config.Config) *VersionManager { s := &VersionManager{} s.cfg = cfg return s diff --git a/pkg/kustomize/apply.go b/pkg/kustomize/apply.go index a1019f17e1..19a0b4bd3a 100644 --- a/pkg/kustomize/apply.go +++ b/pkg/kustomize/apply.go @@ -30,7 +30,7 @@ type Kustomizer struct { kubeconfig string } -func NewKustomizer(cfg *config.MicroshiftConfig) *Kustomizer { +func NewKustomizer(cfg *config.Config) *Kustomizer { return &Kustomizer{ paths: microshiftManifestsDir, kubeconfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 2a5d9a0ad3..3b94100ef6 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -34,7 +34,7 @@ type LoadbalancerServiceController struct { var _ servicemanager.Service = &LoadbalancerServiceController{} -func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { +func NewLoadbalancerServiceController(cfg *config.Config) *LoadbalancerServiceController { return &LoadbalancerServiceController{ NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 3daee405c2..29d7231aaf 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -23,7 +23,7 @@ type MicroShiftmDNSController struct { stopCh chan struct{} } -func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { +func NewMicroShiftmDNSController(cfg *config.Config) *MicroShiftmDNSController { return &MicroShiftmDNSController{ NodeIP: cfg.Node.NodeIP, NodeName: cfg.Node.HostnameOverride, diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index ebc246ded8..76bc0853b8 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -52,7 +52,7 @@ type KubeletServer struct { kubeconfig *kubeletconfig.KubeletConfiguration } -func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { +func NewKubeletServer(cfg *config.Config) *KubeletServer { s := &KubeletServer{} s.configure(cfg) return s @@ -61,7 +61,7 @@ func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { func (s *KubeletServer) Name() string { return componentKubelet } func (s *KubeletServer) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { +func (s *KubeletServer) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("Failed to write kubelet config", err) @@ -94,7 +94,7 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { s.kubeletflags = kubeletFlags } -func (s *KubeletServer) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeletServer) writeConfig(cfg *config.Config) error { certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) servingCertDir := cryptomaterial.KubeletServingCertDir(certsDir) diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e92f9c81d3..a202bc5a04 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -38,7 +38,7 @@ type NetworkConfiguration struct { skipInterfaceConfiguration bool } -func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration { +func NewNetworkConfiguration(cfg *config.Config) *NetworkConfiguration { n := &NetworkConfiguration{} n.configure(cfg) return n @@ -47,7 +47,7 @@ func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration func (n *NetworkConfiguration) Name() string { return componentNetworkConfiguration } func (n *NetworkConfiguration) Dependencies() []string { return []string{} } -func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { +func (n *NetworkConfiguration) configure(cfg *config.Config) { n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index 0c18891ff7..02f43e7018 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -36,7 +36,7 @@ type SysConfWatchController struct { timerFd int } -func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchController { +func NewSysConfWatchController(cfg *config.Config) *SysConfWatchController { // Create a realtime clock timer with asynchronous read support fd, err := unix.TimerfdCreate(unix.CLOCK_REALTIME, unix.TFD_CLOEXEC|unix.TFD_NONBLOCK) if err != nil { From c892acaabdefa4f040e9d1ca53dfc46b191cbda3 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 4 Mar 2023 11:49:32 -0500 Subject: [PATCH 08/10] USHIFT-936: split up large config.go Move data structures and related functions into their own files. --- .../microshift/pkg/config/apiserver.go | 44 ++ .../openshift/microshift/pkg/config/config.go | 488 ------------------ .../microshift/pkg/config/debugging.go | 25 + .../openshift/microshift/pkg/config/dns.go | 14 + .../openshift/microshift/pkg/config/etcd.go | 22 + .../openshift/microshift/pkg/config/files.go | 216 ++++++++ .../openshift/microshift/pkg/config/flags.go | 23 + .../microshift/pkg/config/ingress.go | 6 + .../microshift/pkg/config/kubeconfig.go | 24 + .../microshift/pkg/config/network.go | 63 +++ .../openshift/microshift/pkg/config/node.go | 70 +++ .../openshift/microshift/pkg/config/util.go | 21 + pkg/config/apiserver.go | 44 ++ pkg/config/config.go | 482 ----------------- pkg/config/debugging.go | 25 + pkg/config/debugging_test.go | 48 ++ pkg/config/dns.go | 14 + pkg/config/etcd.go | 22 + pkg/config/files.go | 216 ++++++++ pkg/config/flags.go | 23 + pkg/config/ingress.go | 6 + pkg/config/kubeconfig.go | 24 + pkg/config/network.go | 63 +++ pkg/config/node.go | 70 +++ pkg/config/util.go | 21 + 25 files changed, 1104 insertions(+), 970 deletions(-) create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/files.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/network.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/node.go create mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/util.go create mode 100644 pkg/config/apiserver.go create mode 100644 pkg/config/debugging.go create mode 100644 pkg/config/debugging_test.go create mode 100644 pkg/config/dns.go create mode 100644 pkg/config/etcd.go create mode 100644 pkg/config/files.go create mode 100644 pkg/config/flags.go create mode 100644 pkg/config/ingress.go create mode 100644 pkg/config/kubeconfig.go create mode 100644 pkg/config/network.go create mode 100644 pkg/config/node.go create mode 100644 pkg/config/util.go diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go new file mode 100644 index 0000000000..dcb398c7a0 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go @@ -0,0 +1,44 @@ +package config + +import ( + "net/url" + "strconv" +) + +type ApiServer struct { + // SubjectAltNames added to API server certs + SubjectAltNames []string `json:"subjectAltNames"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` +} + +// extract the api server port from the cluster URL +func (c *Config) ApiServerPort() (int, error) { + var port string + + parsed, err := url.Parse(c.ApiServer.URL) + if err != nil { + return 0, err + } + + // default empty URL to port 6443 + port = parsed.Port() + if port == "" { + port = "6443" + } + portNum, err := strconv.Atoi(port) + if err != nil { + return 0, err + } + return portNum, nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index bc8437a1e9..31aaaca3d3 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -2,80 +2,22 @@ package config import ( "bytes" - "errors" "fmt" - "net" - "net/url" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/mitchellh/go-homedir" - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/component-base/logs" "k8s.io/klog/v2" - ctrl "k8s.io/kubernetes/pkg/controlplane" - "sigs.k8s.io/yaml" "github.com/openshift/microshift/pkg/util" ) const ( - DefaultUserConfigFile = "~/.microshift/config.yaml" - defaultUserDataDir = "~/.microshift/data" - DefaultGlobalConfigFile = "/etc/microshift/config.yaml" - defaultGlobalDataDir = "/var/lib/microshift" - // for files managed via management system in /etc, i.e. user applications - defaultManifestDirEtc = "/etc/microshift/manifests" - // for files embedded in ostree. i.e. cni/other component customizations - defaultManifestDirLib = "/usr/lib/microshift/manifests" - // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) -var ( - configFile = findConfigFile() - dataDir = findDataDir() - manifestsDir = findManifestsDir() -) - -type ClusterConfig struct { - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` -} - -type IngressConfig struct { - ServingCertificate []byte - ServingKey []byte -} - -type EtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string `json:"quotaBackendSize"` - QuotaBackendBytes int64 `json:"-"` - - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string `json:"minDefragSize"` - MinDefragBytes int64 `json:"-"` - MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` - - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string `json:"defragCheckFreq"` - DefragCheckDuration time.Duration `json:"-"` - - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool `json:"doStartupDefrag"` -} - type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` @@ -88,112 +30,6 @@ type Config struct { Ingress IngressConfig `json:"-"` } -type Network struct { - // IP address pool to use for pod IPs. - // This field is immutable after installation. - ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork,omitempty"` - - // IP address pool for services. - // Currently, we only support a single entry here. - // This field is immutable after installation. - ServiceNetwork []string `json:"serviceNetwork,omitempty"` - - // The port range allowed for Services of type NodePort. - // If not specified, the default of 30000-32767 will be used. - // Such Services without a NodePort specified will have one - // automatically allocated from this range. - // This parameter can be updated after the cluster is - // installed. - // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` - - // The DNS server to use - DNS string `json:"-"` -} - -type ClusterNetworkEntry struct { - // The complete block for pod IPs. - CIDR string `json:"cidr,omitempty"` -} - -type DNS struct { - // baseDomain is the base domain of the cluster. All managed DNS records will - // be sub-domains of this base. - // - // For example, given the base domain `example.com`, router exposed - // domains will be formed as `*.apps.example.com` by default, - // and API service will have a DNS entry for `api.example.com`, - // as well as "api-int.example.com" for internal k8s API access. - // - // Once set, this field cannot be changed. - BaseDomain string `json:"baseDomain"` -} - -type ApiServer struct { - // SubjectAltNames added to API server certs - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - // Determines if kube-apiserver controller should configure the - // AdvertiseAddress in the loopback interface. Automatically computed. - SkipInterface bool `json:"-"` - - // The URL of the API server - URL string `json:"-"` -} - -type Node struct { - // If non-empty, will use this string to identify the node instead of the hostname - HostnameOverride string `json:"hostnameOverride"` - - // IP address of the node, passed to the kubelet. - // If not specified, kubelet will use the node's default IP address. - NodeIP string `json:"nodeIP"` -} - -type Debugging struct { - // Valid values are: "Normal", "Debug", "Trace", "TraceAll". - // Defaults to "Normal". - LogLevel string `json:"logLevel"` -} - -func GetConfigFile() string { - return configFile -} - -func GetDataDir() string { - return dataDir -} - -func GetManifestsDir() []string { - return manifestsDir -} - -// KubeConfigID identifies the different kubeconfigs managed in the DataDir -type KubeConfigID string - -const ( - KubeAdmin KubeConfigID = "kubeadmin" - KubeControllerManager KubeConfigID = "kube-controller-manager" - KubeScheduler KubeConfigID = "kube-scheduler" - Kubelet KubeConfigID = "kubelet" - ClusterPolicyController KubeConfigID = "cluster-policy-controller" - RouteControllerManager KubeConfigID = "route-controller-manager" -) - -// KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *Config) KubeConfigPath(id KubeConfigID) string { - return filepath.Join(dataDir, "resources", string(id), "kubeconfig") -} - -func (cfg *Config) KubeConfigAdminPath(id string) string { - return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") -} - func getAllHostnames() ([]string, error) { cmd := exec.Command("/bin/hostname", "-A") var out bytes.Buffer @@ -263,327 +99,3 @@ func NewMicroshiftConfig() *Config { }, } } - -// Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *Config) isDefaultNodeName() bool { - hostname, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - return c.Node.HostnameOverride == hostname -} - -// Read or set the NodeName that will be used for this MicroShift instance -func (c *Config) establishNodeName() (string, error) { - filePath := filepath.Join(GetDataDir(), ".nodename") - contents, err := os.ReadFile(filePath) - if os.IsNotExist(err) { - // ensure that dataDir exists - os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { - return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) - } - return c.Node.HostnameOverride, nil - } else if err != nil { - return "", err - } - return string(contents), nil -} - -// Validate the NodeName to be used for this MicroShift instances -func (c *Config) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) - } - - establishedNodeName, err := c.establishNodeName() - if err != nil { - return fmt.Errorf("failed to establish NodeName: %v", err) - } - - if establishedNodeName != c.Node.HostnameOverride { - if !isDefaultNodeName { - return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.Node.HostnameOverride, establishedNodeName) - } else { - c.Node.HostnameOverride = establishedNodeName - klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) - } - } - - return nil -} - -// extract the api server port from the cluster URL -func (c *Config) ApiServerPort() (int, error) { - var port string - - parsed, err := url.Parse(c.ApiServer.URL) - if err != nil { - return 0, err - } - - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - return portNum, nil -} - -// Returns the default user config file if that exists, else the default global -// config file, else the empty string. -func findConfigFile() string { - userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) - if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { - if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { - return "" - } else { - return DefaultGlobalConfigFile - } - } else { - return userConfigFile - } -} - -// Returns the default user data dir if it exists or the user is non-root. -// Returns the default global data dir otherwise. -func findDataDir() string { - userDataDir, _ := homedir.Expand(defaultUserDataDir) - if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { - if os.Geteuid() > 0 { - return userDataDir - } else { - return defaultGlobalDataDir - } - } else { - return userDataDir - } -} - -// Returns the default manifests directories -func findManifestsDir() []string { - var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} - return manifestsDir -} - -func StringInList(s string, list []string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} - -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) - } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) - } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - c.Etcd = config.Etcd - if c.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) - if err != nil { - return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) - } - c.Etcd.DefragCheckDuration = d - } - if c.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) - if err != nil { - return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) - } - if !q.IsZero() { - c.Etcd.MinDefragBytes = q.Value() - } - } - if c.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) - if err != nil { - return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) - } - if !q.IsZero() { - c.Etcd.QuotaBackendBytes = q.Value() - } - } - - return nil -} - -func (c *Config) computeAndUpdateClusterDNS() error { - if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") - } - - clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) - } - c.Network.DNS = clusterDNS - return nil -} - -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then grab it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) - if err != nil { - return fmt.Errorf("error getting apiserver IP: %v", err) - } - c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true - } - - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } - } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) - } - - return nil -} - -// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork -func getClusterDNS(serviceCIDR string) (string, error) { - _, service, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) - } - dnsClusterIP, err := cidr.Host(service, 10) - if err != nil { - return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) - } - - return dnsClusterIP.String(), nil -} - -func stringSliceContains(list []string, elements ...string) bool { - for _, value := range list { - for _, element := range elements { - if value == element { - return true - } - } - } - return false -} - -// GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { - var verbosity int - switch c.Debugging.LogLevel { - case "Normal": - verbosity = 2 - case "Debug": - verbosity = 4 - case "Trace": - verbosity = 6 - case "TraceAll": - verbosity = 8 - default: - verbosity = 2 - } - return verbosity -} - -func HideUnsupportedFlags(flags *pflag.FlagSet) { - // hide logging flags that we do not use/support - loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) - logs.AddFlags(loggingFlags) - - supportedLoggingFlags := sets.NewString("v") - - loggingFlags.VisitAll(func(pf *pflag.Flag) { - if !supportedLoggingFlags.Has(pf.Name) { - flags.MarkHidden(pf.Name) - } - }) - - flags.MarkHidden("version") -} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go new file mode 100644 index 0000000000..a540f5490c --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go @@ -0,0 +1,25 @@ +package config + +type Debugging struct { + // Valid values are: "Normal", "Debug", "Trace", "TraceAll". + // Defaults to "Normal". + LogLevel string `json:"logLevel"` +} + +// GetVerbosity returns the numerical value for LogLevel which is an enum +func (c *Config) GetVerbosity() int { + var verbosity int + switch c.Debugging.LogLevel { + case "Normal": + verbosity = 2 + case "Debug": + verbosity = 4 + case "Trace": + verbosity = 6 + case "TraceAll": + verbosity = 8 + default: + verbosity = 2 + } + return verbosity +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go new file mode 100644 index 0000000000..ca6f11e88e --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go @@ -0,0 +1,14 @@ +package config + +type DNS struct { + // baseDomain is the base domain of the cluster. All managed DNS records will + // be sub-domains of this base. + // + // For example, given the base domain `example.com`, router exposed + // domains will be formed as `*.apps.example.com` by default, + // and API service will have a DNS entry for `api.example.com`, + // as well as "api-int.example.com" for internal k8s API access. + // + // Once set, this field cannot be changed. + BaseDomain string `json:"baseDomain"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go new file mode 100644 index 0000000000..27917eab97 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go @@ -0,0 +1,22 @@ +package config + +import "time" + +type EtcdConfig struct { + // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value + QuotaBackendSize string `json:"quotaBackendSize"` + QuotaBackendBytes int64 `json:"-"` + + // If the backend is fragmented more than `maxFragmentedPercentage` + // and the database size is greater than `minDefragSize`, do a defrag. + MinDefragSize string `json:"minDefragSize"` + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` + + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). + DefragCheckFreq string `json:"defragCheckFreq"` + DefragCheckDuration time.Duration `json:"-"` + + // Whether or not to do a defrag when the server finishes starting + DoStartupDefrag bool `json:"doStartupDefrag"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go new file mode 100644 index 0000000000..1253fe7b46 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -0,0 +1,216 @@ +package config + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "time" + + "github.com/mitchellh/go-homedir" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog/v2" + ctrl "k8s.io/kubernetes/pkg/controlplane" + "sigs.k8s.io/yaml" +) + +const ( + DefaultUserConfigFile = "~/.microshift/config.yaml" + defaultUserDataDir = "~/.microshift/data" + DefaultGlobalConfigFile = "/etc/microshift/config.yaml" + defaultGlobalDataDir = "/var/lib/microshift" + // for files managed via management system in /etc, i.e. user applications + defaultManifestDirEtc = "/etc/microshift/manifests" + // for files embedded in ostree. i.e. cni/other component customizations + defaultManifestDirLib = "/usr/lib/microshift/manifests" +) + +var ( + configFile = findConfigFile() + dataDir = findDataDir() + manifestsDir = findManifestsDir() +) + +func GetConfigFile() string { + return configFile +} + +func GetDataDir() string { + return dataDir +} + +func GetManifestsDir() []string { + return manifestsDir +} + +// Returns the default user config file if that exists, else the default global +// config file, else the empty string. +func findConfigFile() string { + userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) + if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { + return "" + } else { + return DefaultGlobalConfigFile + } + } else { + return userConfigFile + } +} + +// Returns the default user data dir if it exists or the user is non-root. +// Returns the default global data dir otherwise. +func findDataDir() string { + userDataDir, _ := homedir.Expand(defaultUserDataDir) + if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { + if os.Geteuid() > 0 { + return userDataDir + } else { + return defaultGlobalDataDir + } + } else { + return userDataDir + } +} + +// Returns the default manifests directories +func findManifestsDir() []string { + var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} + return manifestsDir +} + +func (c *Config) ReadFromConfigFile(configFile string) error { + contents, err := os.ReadFile(configFile) + if err != nil { + return fmt.Errorf("reading config file %q: %v", configFile, err) + } + var config Config + if err := yaml.Unmarshal(contents, &config); err != nil { + return fmt.Errorf("decoding config file %s: %v", configFile, err) + } + + // Wire new Config type to existing Config + c.Node = config.Node + c.Debugging = config.Debugging + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + c.DNS = config.DNS + c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" + + c.Etcd = config.Etcd + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) + if err != nil { + return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) + } + c.Etcd.DefragCheckDuration = d + } + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) + if err != nil { + return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) + } + if !q.IsZero() { + c.Etcd.MinDefragBytes = q.Value() + } + } + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) + if err != nil { + return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) + } + if !q.IsZero() { + c.Etcd.QuotaBackendBytes = q.Value() + } + } + + return nil +} + +// Note: add a configFile parameter here because of unit test requiring custom +// local directory +func (c *Config) ReadAndValidate(configFile string) error { + if configFile != "" { + if err := c.ReadFromConfigFile(configFile); err != nil { + return err + } + } + + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + // If KAS advertise address is not configured then grab it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) + if err != nil { + return fmt.Errorf("error getting apiserver IP: %v", err) + } + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { + klog.Fatalf("Error in validating node name: %v", err) + } + + return nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go new file mode 100644 index 0000000000..e15ee75a1c --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/logs" +) + +func HideUnsupportedFlags(flags *pflag.FlagSet) { + // hide logging flags that we do not use/support + loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) + logs.AddFlags(loggingFlags) + + supportedLoggingFlags := sets.NewString("v") + + loggingFlags.VisitAll(func(pf *pflag.Flag) { + if !supportedLoggingFlags.Has(pf.Name) { + flags.MarkHidden(pf.Name) + } + }) + + flags.MarkHidden("version") +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go new file mode 100644 index 0000000000..a8980ae1ed --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go @@ -0,0 +1,6 @@ +package config + +type IngressConfig struct { + ServingCertificate []byte + ServingKey []byte +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go new file mode 100644 index 0000000000..1e33b07b96 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go @@ -0,0 +1,24 @@ +package config + +import "path/filepath" + +// KubeConfigID identifies the different kubeconfigs managed in the DataDir +type KubeConfigID string + +const ( + KubeAdmin KubeConfigID = "kubeadmin" + KubeControllerManager KubeConfigID = "kube-controller-manager" + KubeScheduler KubeConfigID = "kube-scheduler" + Kubelet KubeConfigID = "kubelet" + ClusterPolicyController KubeConfigID = "cluster-policy-controller" + RouteControllerManager KubeConfigID = "route-controller-manager" +) + +// KubeConfigPath returns the path to the specified kubeconfig file. +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { + return filepath.Join(dataDir, "resources", string(id), "kubeconfig") +} + +func (cfg *Config) KubeConfigAdminPath(id string) string { + return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go new file mode 100644 index 0000000000..f7e04e5838 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" + "net" + + "github.com/apparentlymart/go-cidr/cidr" +) + +type Network struct { + // IP address pool to use for pod IPs. + // This field is immutable after installation. + ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork"` + + // IP address pool for services. + // Currently, we only support a single entry here. + // This field is immutable after installation. + ServiceNetwork []string `json:"serviceNetwork"` + + // The port range allowed for Services of type NodePort. + // If not specified, the default of 30000-32767 will be used. + // Such Services without a NodePort specified will have one + // automatically allocated from this range. + // This parameter can be updated after the cluster is + // installed. + // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` + ServiceNodePortRange string `json:"serviceNodePortRange"` + + // The DNS server to use + DNS string `json:"-"` +} + +type ClusterNetworkEntry struct { + // The complete block for pod IPs. + CIDR string `json:"cidr"` +} + +func (c *Config) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + +// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork +func getClusterDNS(serviceCIDR string) (string, error) { + _, service, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) + } + dnsClusterIP, err := cidr.Host(service, 10) + if err != nil { + return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) + } + + return dnsClusterIP.String(), nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go new file mode 100644 index 0000000000..1622be3ab4 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go @@ -0,0 +1,70 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + + "k8s.io/klog/v2" +) + +type Node struct { + // If non-empty, will use this string to identify the node instead of the hostname + HostnameOverride string `json:"hostnameOverride"` + + // IP address of the node, passed to the kubelet. + // If not specified, kubelet will use the node's default IP address. + NodeIP string `json:"nodeIP"` +} + +// Determine if the config file specified a NodeName (by default it's assigned the hostname) +func (c *Config) isDefaultNodeName() bool { + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Failed to get hostname %v", err) + } + return c.Node.HostnameOverride == hostname +} + +// Read or set the NodeName that will be used for this MicroShift instance +func (c *Config) establishNodeName() (string, error) { + filePath := filepath.Join(GetDataDir(), ".nodename") + contents, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + // ensure that dataDir exists + os.MkdirAll(GetDataDir(), 0700) + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { + return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) + } + return c.Node.HostnameOverride, nil + } else if err != nil { + return "", err + } + return string(contents), nil +} + +// Validate the NodeName to be used for this MicroShift instances +func (c *Config) validateNodeName(isDefaultNodeName bool) error { + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) + } + + establishedNodeName, err := c.establishNodeName() + if err != nil { + return fmt.Errorf("failed to establish NodeName: %v", err) + } + + if establishedNodeName != c.Node.HostnameOverride { + if !isDefaultNodeName { + return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", + c.Node.HostnameOverride, establishedNodeName) + } else { + c.Node.HostnameOverride = establishedNodeName + klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) + } + } + + return nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go new file mode 100644 index 0000000000..6bb69a5079 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go @@ -0,0 +1,21 @@ +package config + +func StringInList(s string, list []string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func stringSliceContains(list []string, elements ...string) bool { + for _, value := range list { + for _, element := range elements { + if value == element { + return true + } + } + } + return false +} diff --git a/pkg/config/apiserver.go b/pkg/config/apiserver.go new file mode 100644 index 0000000000..dcb398c7a0 --- /dev/null +++ b/pkg/config/apiserver.go @@ -0,0 +1,44 @@ +package config + +import ( + "net/url" + "strconv" +) + +type ApiServer struct { + // SubjectAltNames added to API server certs + SubjectAltNames []string `json:"subjectAltNames"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` +} + +// extract the api server port from the cluster URL +func (c *Config) ApiServerPort() (int, error) { + var port string + + parsed, err := url.Parse(c.ApiServer.URL) + if err != nil { + return 0, err + } + + // default empty URL to port 6443 + port = parsed.Port() + if port == "" { + port = "6443" + } + portNum, err := strconv.Atoi(port) + if err != nil { + return 0, err + } + return portNum, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 5f969305c1..31aaaca3d3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,74 +2,22 @@ package config import ( "bytes" - "errors" "fmt" - "net" - "net/url" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/mitchellh/go-homedir" - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/component-base/logs" "k8s.io/klog/v2" - ctrl "k8s.io/kubernetes/pkg/controlplane" - "sigs.k8s.io/yaml" "github.com/openshift/microshift/pkg/util" ) const ( - DefaultUserConfigFile = "~/.microshift/config.yaml" - defaultUserDataDir = "~/.microshift/data" - DefaultGlobalConfigFile = "/etc/microshift/config.yaml" - defaultGlobalDataDir = "/var/lib/microshift" - // for files managed via management system in /etc, i.e. user applications - defaultManifestDirEtc = "/etc/microshift/manifests" - // for files embedded in ostree. i.e. cni/other component customizations - defaultManifestDirLib = "/usr/lib/microshift/manifests" - // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) -var ( - configFile = findConfigFile() - dataDir = findDataDir() - manifestsDir = findManifestsDir() -) - -type IngressConfig struct { - ServingCertificate []byte - ServingKey []byte -} - -type EtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string `json:"quotaBackendSize"` - QuotaBackendBytes int64 `json:"-"` - - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string `json:"minDefragSize"` - MinDefragBytes int64 `json:"-"` - MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` - - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string `json:"defragCheckFreq"` - DefragCheckDuration time.Duration `json:"-"` - - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool `json:"doStartupDefrag"` -} - type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` @@ -82,112 +30,6 @@ type Config struct { Ingress IngressConfig `json:"-"` } -type Network struct { - // IP address pool to use for pod IPs. - // This field is immutable after installation. - ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork,omitempty"` - - // IP address pool for services. - // Currently, we only support a single entry here. - // This field is immutable after installation. - ServiceNetwork []string `json:"serviceNetwork,omitempty"` - - // The port range allowed for Services of type NodePort. - // If not specified, the default of 30000-32767 will be used. - // Such Services without a NodePort specified will have one - // automatically allocated from this range. - // This parameter can be updated after the cluster is - // installed. - // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` - - // The DNS server to use - DNS string `json:"-"` -} - -type ClusterNetworkEntry struct { - // The complete block for pod IPs. - CIDR string `json:"cidr,omitempty"` -} - -type DNS struct { - // baseDomain is the base domain of the cluster. All managed DNS records will - // be sub-domains of this base. - // - // For example, given the base domain `example.com`, router exposed - // domains will be formed as `*.apps.example.com` by default, - // and API service will have a DNS entry for `api.example.com`, - // as well as "api-int.example.com" for internal k8s API access. - // - // Once set, this field cannot be changed. - BaseDomain string `json:"baseDomain"` -} - -type ApiServer struct { - // SubjectAltNames added to API server certs - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - // Determines if kube-apiserver controller should configure the - // AdvertiseAddress in the loopback interface. Automatically computed. - SkipInterface bool `json:"-"` - - // The URL of the API server - URL string `json:"-"` -} - -type Node struct { - // If non-empty, will use this string to identify the node instead of the hostname - HostnameOverride string `json:"hostnameOverride"` - - // IP address of the node, passed to the kubelet. - // If not specified, kubelet will use the node's default IP address. - NodeIP string `json:"nodeIP"` -} - -type Debugging struct { - // Valid values are: "Normal", "Debug", "Trace", "TraceAll". - // Defaults to "Normal". - LogLevel string `json:"logLevel"` -} - -func GetConfigFile() string { - return configFile -} - -func GetDataDir() string { - return dataDir -} - -func GetManifestsDir() []string { - return manifestsDir -} - -// KubeConfigID identifies the different kubeconfigs managed in the DataDir -type KubeConfigID string - -const ( - KubeAdmin KubeConfigID = "kubeadmin" - KubeControllerManager KubeConfigID = "kube-controller-manager" - KubeScheduler KubeConfigID = "kube-scheduler" - Kubelet KubeConfigID = "kubelet" - ClusterPolicyController KubeConfigID = "cluster-policy-controller" - RouteControllerManager KubeConfigID = "route-controller-manager" -) - -// KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *Config) KubeConfigPath(id KubeConfigID) string { - return filepath.Join(dataDir, "resources", string(id), "kubeconfig") -} - -func (cfg *Config) KubeConfigAdminPath(id string) string { - return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") -} - func getAllHostnames() ([]string, error) { cmd := exec.Command("/bin/hostname", "-A") var out bytes.Buffer @@ -257,327 +99,3 @@ func NewMicroshiftConfig() *Config { }, } } - -// Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *Config) isDefaultNodeName() bool { - hostname, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - return c.Node.HostnameOverride == hostname -} - -// Read or set the NodeName that will be used for this MicroShift instance -func (c *Config) establishNodeName() (string, error) { - filePath := filepath.Join(GetDataDir(), ".nodename") - contents, err := os.ReadFile(filePath) - if os.IsNotExist(err) { - // ensure that dataDir exists - os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { - return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) - } - return c.Node.HostnameOverride, nil - } else if err != nil { - return "", err - } - return string(contents), nil -} - -// Validate the NodeName to be used for this MicroShift instances -func (c *Config) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) - } - - establishedNodeName, err := c.establishNodeName() - if err != nil { - return fmt.Errorf("failed to establish NodeName: %v", err) - } - - if establishedNodeName != c.Node.HostnameOverride { - if !isDefaultNodeName { - return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.Node.HostnameOverride, establishedNodeName) - } else { - c.Node.HostnameOverride = establishedNodeName - klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) - } - } - - return nil -} - -// extract the api server port from the cluster URL -func (c *Config) ApiServerPort() (int, error) { - var port string - - parsed, err := url.Parse(c.ApiServer.URL) - if err != nil { - return 0, err - } - - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err - } - return portNum, nil -} - -// Returns the default user config file if that exists, else the default global -// config file, else the empty string. -func findConfigFile() string { - userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) - if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { - if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { - return "" - } else { - return DefaultGlobalConfigFile - } - } else { - return userConfigFile - } -} - -// Returns the default user data dir if it exists or the user is non-root. -// Returns the default global data dir otherwise. -func findDataDir() string { - userDataDir, _ := homedir.Expand(defaultUserDataDir) - if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { - if os.Geteuid() > 0 { - return userDataDir - } else { - return defaultGlobalDataDir - } - } else { - return userDataDir - } -} - -// Returns the default manifests directories -func findManifestsDir() []string { - var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} - return manifestsDir -} - -func StringInList(s string, list []string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} - -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) - } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) - } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - c.Etcd = config.Etcd - if c.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) - if err != nil { - return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) - } - c.Etcd.DefragCheckDuration = d - } - if c.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) - if err != nil { - return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) - } - if !q.IsZero() { - c.Etcd.MinDefragBytes = q.Value() - } - } - if c.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) - if err != nil { - return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) - } - if !q.IsZero() { - c.Etcd.QuotaBackendBytes = q.Value() - } - } - - return nil -} - -func (c *Config) computeAndUpdateClusterDNS() error { - if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") - } - - clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) - } - c.Network.DNS = clusterDNS - return nil -} - -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then grab it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) - if err != nil { - return fmt.Errorf("error getting apiserver IP: %v", err) - } - c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true - } - - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } - } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) - } - - return nil -} - -// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork -func getClusterDNS(serviceCIDR string) (string, error) { - _, service, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) - } - dnsClusterIP, err := cidr.Host(service, 10) - if err != nil { - return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) - } - - return dnsClusterIP.String(), nil -} - -func stringSliceContains(list []string, elements ...string) bool { - for _, value := range list { - for _, element := range elements { - if value == element { - return true - } - } - } - return false -} - -// GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { - var verbosity int - switch c.Debugging.LogLevel { - case "Normal": - verbosity = 2 - case "Debug": - verbosity = 4 - case "Trace": - verbosity = 6 - case "TraceAll": - verbosity = 8 - default: - verbosity = 2 - } - return verbosity -} - -func HideUnsupportedFlags(flags *pflag.FlagSet) { - // hide logging flags that we do not use/support - loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) - logs.AddFlags(loggingFlags) - - supportedLoggingFlags := sets.NewString("v") - - loggingFlags.VisitAll(func(pf *pflag.Flag) { - if !supportedLoggingFlags.Has(pf.Name) { - flags.MarkHidden(pf.Name) - } - }) - - flags.MarkHidden("version") -} diff --git a/pkg/config/debugging.go b/pkg/config/debugging.go new file mode 100644 index 0000000000..a540f5490c --- /dev/null +++ b/pkg/config/debugging.go @@ -0,0 +1,25 @@ +package config + +type Debugging struct { + // Valid values are: "Normal", "Debug", "Trace", "TraceAll". + // Defaults to "Normal". + LogLevel string `json:"logLevel"` +} + +// GetVerbosity returns the numerical value for LogLevel which is an enum +func (c *Config) GetVerbosity() int { + var verbosity int + switch c.Debugging.LogLevel { + case "Normal": + verbosity = 2 + case "Debug": + verbosity = 4 + case "Trace": + verbosity = 6 + case "TraceAll": + verbosity = 8 + default: + verbosity = 2 + } + return verbosity +} diff --git a/pkg/config/debugging_test.go b/pkg/config/debugging_test.go new file mode 100644 index 0000000000..2a934370e4 --- /dev/null +++ b/pkg/config/debugging_test.go @@ -0,0 +1,48 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetVerbosity(t *testing.T) { + var ttests = []struct { + setting string + level int + }{ + { + setting: "Normal", + level: 2, + }, + { + setting: "Debug", + level: 4, + }, + { + setting: "Trace", + level: 6, + }, + { + setting: "TraceAll", + level: 8, + }, + { + setting: "Unknown", + level: 2, + }, + { + setting: "", + level: 2, + }, + } + + for _, tt := range ttests { + t.Run(tt.setting, func(t *testing.T) { + config := NewMicroshiftConfig() + config.Debugging.LogLevel = tt.setting + verbosity := config.GetVerbosity() + assert.Equal(t, tt.level, verbosity) + }) + } +} diff --git a/pkg/config/dns.go b/pkg/config/dns.go new file mode 100644 index 0000000000..ca6f11e88e --- /dev/null +++ b/pkg/config/dns.go @@ -0,0 +1,14 @@ +package config + +type DNS struct { + // baseDomain is the base domain of the cluster. All managed DNS records will + // be sub-domains of this base. + // + // For example, given the base domain `example.com`, router exposed + // domains will be formed as `*.apps.example.com` by default, + // and API service will have a DNS entry for `api.example.com`, + // as well as "api-int.example.com" for internal k8s API access. + // + // Once set, this field cannot be changed. + BaseDomain string `json:"baseDomain"` +} diff --git a/pkg/config/etcd.go b/pkg/config/etcd.go new file mode 100644 index 0000000000..27917eab97 --- /dev/null +++ b/pkg/config/etcd.go @@ -0,0 +1,22 @@ +package config + +import "time" + +type EtcdConfig struct { + // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value + QuotaBackendSize string `json:"quotaBackendSize"` + QuotaBackendBytes int64 `json:"-"` + + // If the backend is fragmented more than `maxFragmentedPercentage` + // and the database size is greater than `minDefragSize`, do a defrag. + MinDefragSize string `json:"minDefragSize"` + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` + + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). + DefragCheckFreq string `json:"defragCheckFreq"` + DefragCheckDuration time.Duration `json:"-"` + + // Whether or not to do a defrag when the server finishes starting + DoStartupDefrag bool `json:"doStartupDefrag"` +} diff --git a/pkg/config/files.go b/pkg/config/files.go new file mode 100644 index 0000000000..1253fe7b46 --- /dev/null +++ b/pkg/config/files.go @@ -0,0 +1,216 @@ +package config + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "time" + + "github.com/mitchellh/go-homedir" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/klog/v2" + ctrl "k8s.io/kubernetes/pkg/controlplane" + "sigs.k8s.io/yaml" +) + +const ( + DefaultUserConfigFile = "~/.microshift/config.yaml" + defaultUserDataDir = "~/.microshift/data" + DefaultGlobalConfigFile = "/etc/microshift/config.yaml" + defaultGlobalDataDir = "/var/lib/microshift" + // for files managed via management system in /etc, i.e. user applications + defaultManifestDirEtc = "/etc/microshift/manifests" + // for files embedded in ostree. i.e. cni/other component customizations + defaultManifestDirLib = "/usr/lib/microshift/manifests" +) + +var ( + configFile = findConfigFile() + dataDir = findDataDir() + manifestsDir = findManifestsDir() +) + +func GetConfigFile() string { + return configFile +} + +func GetDataDir() string { + return dataDir +} + +func GetManifestsDir() []string { + return manifestsDir +} + +// Returns the default user config file if that exists, else the default global +// config file, else the empty string. +func findConfigFile() string { + userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) + if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { + return "" + } else { + return DefaultGlobalConfigFile + } + } else { + return userConfigFile + } +} + +// Returns the default user data dir if it exists or the user is non-root. +// Returns the default global data dir otherwise. +func findDataDir() string { + userDataDir, _ := homedir.Expand(defaultUserDataDir) + if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { + if os.Geteuid() > 0 { + return userDataDir + } else { + return defaultGlobalDataDir + } + } else { + return userDataDir + } +} + +// Returns the default manifests directories +func findManifestsDir() []string { + var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} + return manifestsDir +} + +func (c *Config) ReadFromConfigFile(configFile string) error { + contents, err := os.ReadFile(configFile) + if err != nil { + return fmt.Errorf("reading config file %q: %v", configFile, err) + } + var config Config + if err := yaml.Unmarshal(contents, &config); err != nil { + return fmt.Errorf("decoding config file %s: %v", configFile, err) + } + + // Wire new Config type to existing Config + c.Node = config.Node + c.Debugging = config.Debugging + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + c.DNS = config.DNS + c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" + + c.Etcd = config.Etcd + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) + if err != nil { + return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) + } + c.Etcd.DefragCheckDuration = d + } + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) + if err != nil { + return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) + } + if !q.IsZero() { + c.Etcd.MinDefragBytes = q.Value() + } + } + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) + if err != nil { + return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) + } + if !q.IsZero() { + c.Etcd.QuotaBackendBytes = q.Value() + } + } + + return nil +} + +// Note: add a configFile parameter here because of unit test requiring custom +// local directory +func (c *Config) ReadAndValidate(configFile string) error { + if configFile != "" { + if err := c.ReadFromConfigFile(configFile); err != nil { + return err + } + } + + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + } + + // If KAS advertise address is not configured then grab it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) + if err != nil { + return fmt.Errorf("error getting apiserver IP: %v", err) + } + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { + klog.Fatalf("Error in validating node name: %v", err) + } + + return nil +} diff --git a/pkg/config/flags.go b/pkg/config/flags.go new file mode 100644 index 0000000000..e15ee75a1c --- /dev/null +++ b/pkg/config/flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/logs" +) + +func HideUnsupportedFlags(flags *pflag.FlagSet) { + // hide logging flags that we do not use/support + loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) + logs.AddFlags(loggingFlags) + + supportedLoggingFlags := sets.NewString("v") + + loggingFlags.VisitAll(func(pf *pflag.Flag) { + if !supportedLoggingFlags.Has(pf.Name) { + flags.MarkHidden(pf.Name) + } + }) + + flags.MarkHidden("version") +} diff --git a/pkg/config/ingress.go b/pkg/config/ingress.go new file mode 100644 index 0000000000..a8980ae1ed --- /dev/null +++ b/pkg/config/ingress.go @@ -0,0 +1,6 @@ +package config + +type IngressConfig struct { + ServingCertificate []byte + ServingKey []byte +} diff --git a/pkg/config/kubeconfig.go b/pkg/config/kubeconfig.go new file mode 100644 index 0000000000..1e33b07b96 --- /dev/null +++ b/pkg/config/kubeconfig.go @@ -0,0 +1,24 @@ +package config + +import "path/filepath" + +// KubeConfigID identifies the different kubeconfigs managed in the DataDir +type KubeConfigID string + +const ( + KubeAdmin KubeConfigID = "kubeadmin" + KubeControllerManager KubeConfigID = "kube-controller-manager" + KubeScheduler KubeConfigID = "kube-scheduler" + Kubelet KubeConfigID = "kubelet" + ClusterPolicyController KubeConfigID = "cluster-policy-controller" + RouteControllerManager KubeConfigID = "route-controller-manager" +) + +// KubeConfigPath returns the path to the specified kubeconfig file. +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { + return filepath.Join(dataDir, "resources", string(id), "kubeconfig") +} + +func (cfg *Config) KubeConfigAdminPath(id string) string { + return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") +} diff --git a/pkg/config/network.go b/pkg/config/network.go new file mode 100644 index 0000000000..f7e04e5838 --- /dev/null +++ b/pkg/config/network.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" + "net" + + "github.com/apparentlymart/go-cidr/cidr" +) + +type Network struct { + // IP address pool to use for pod IPs. + // This field is immutable after installation. + ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork"` + + // IP address pool for services. + // Currently, we only support a single entry here. + // This field is immutable after installation. + ServiceNetwork []string `json:"serviceNetwork"` + + // The port range allowed for Services of type NodePort. + // If not specified, the default of 30000-32767 will be used. + // Such Services without a NodePort specified will have one + // automatically allocated from this range. + // This parameter can be updated after the cluster is + // installed. + // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` + ServiceNodePortRange string `json:"serviceNodePortRange"` + + // The DNS server to use + DNS string `json:"-"` +} + +type ClusterNetworkEntry struct { + // The complete block for pod IPs. + CIDR string `json:"cidr"` +} + +func (c *Config) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + +// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork +func getClusterDNS(serviceCIDR string) (string, error) { + _, service, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) + } + dnsClusterIP, err := cidr.Host(service, 10) + if err != nil { + return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) + } + + return dnsClusterIP.String(), nil +} diff --git a/pkg/config/node.go b/pkg/config/node.go new file mode 100644 index 0000000000..1622be3ab4 --- /dev/null +++ b/pkg/config/node.go @@ -0,0 +1,70 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + + "k8s.io/klog/v2" +) + +type Node struct { + // If non-empty, will use this string to identify the node instead of the hostname + HostnameOverride string `json:"hostnameOverride"` + + // IP address of the node, passed to the kubelet. + // If not specified, kubelet will use the node's default IP address. + NodeIP string `json:"nodeIP"` +} + +// Determine if the config file specified a NodeName (by default it's assigned the hostname) +func (c *Config) isDefaultNodeName() bool { + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Failed to get hostname %v", err) + } + return c.Node.HostnameOverride == hostname +} + +// Read or set the NodeName that will be used for this MicroShift instance +func (c *Config) establishNodeName() (string, error) { + filePath := filepath.Join(GetDataDir(), ".nodename") + contents, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + // ensure that dataDir exists + os.MkdirAll(GetDataDir(), 0700) + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { + return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) + } + return c.Node.HostnameOverride, nil + } else if err != nil { + return "", err + } + return string(contents), nil +} + +// Validate the NodeName to be used for this MicroShift instances +func (c *Config) validateNodeName(isDefaultNodeName bool) error { + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) + } + + establishedNodeName, err := c.establishNodeName() + if err != nil { + return fmt.Errorf("failed to establish NodeName: %v", err) + } + + if establishedNodeName != c.Node.HostnameOverride { + if !isDefaultNodeName { + return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", + c.Node.HostnameOverride, establishedNodeName) + } else { + c.Node.HostnameOverride = establishedNodeName + klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) + } + } + + return nil +} diff --git a/pkg/config/util.go b/pkg/config/util.go new file mode 100644 index 0000000000..6bb69a5079 --- /dev/null +++ b/pkg/config/util.go @@ -0,0 +1,21 @@ +package config + +func StringInList(s string, list []string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func stringSliceContains(list []string, elements ...string) bool { + for _, value := range list { + for _, element := range elements { + if value == element { + return true + } + } + } + return false +} From 92a5f0ed19059a467213001ee4594289ca0ed94b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 4 Mar 2023 12:44:26 -0500 Subject: [PATCH 09/10] USHIFT-936: refactor reading and validating of config This change streamlines the reading and validation functions so that it is easier to get a Config instance with the defaults or with valid values read from a file. A new convenience function for loading the active configuration, ignoring a missing configuration file, is added. The k8s YAML parser is used to read the config file, instead of the default parser, to ensure support for `json:"-"` directives to ignore fields in the input. The unit tests are updated to parse input config instead of using a data structure. This more accurately simulates what happens when the real program reads the config if only part of the file is populated, and clarifies which part of the config each test scenario is checking. It also allows for more, smaller, scenarios to isolate configuration validation and defaulting failures. The validation for node name changing is only relevant when starting the server, so that is moved to the run command implementation to make the unit tests for config easier to understand. The show-config command implementation no longer needs to map between multiple configuration data structures, so streamline that code. --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 239 +++++++++--- .../openshift/microshift/pkg/config/files.go | 163 ++------ .../microshift/pkg/config/network.go | 9 +- .../openshift/microshift/pkg/config/node.go | 7 + pkg/cmd/run.go | 16 +- pkg/cmd/showConfig.go | 38 +- pkg/config/config.go | 239 +++++++++--- pkg/config/config_test.go | 351 ++++++++---------- pkg/config/files.go | 163 ++------ pkg/config/network.go | 9 +- pkg/config/node.go | 7 + 12 files changed, 632 insertions(+), 613 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 67f3cb9a41..146d7bc838 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -22,11 +22,11 @@ import ( ) func NewRunEtcdCommand() *cobra.Command { - cfg := config.NewMicroshiftConfig() cmd := &cobra.Command{ Use: "run", RunE: func(cmd *cobra.Command, args []string) (err error) { - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { + cfg, err := config.GetActiveConfig() + if err != nil { klog.Fatalf("Error in reading and validating MicroShift config: %v", err) } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 31aaaca3d3..fb7243a797 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -3,18 +3,23 @@ package config import ( "bytes" "fmt" + "net" + "net/url" "os" "os/exec" "strings" "time" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + ctrl "k8s.io/kubernetes/pkg/controlplane" "github.com/openshift/microshift/pkg/util" ) const ( + // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) @@ -30,6 +35,186 @@ type Config struct { Ingress IngressConfig `json:"-"` } +func NewMicroshiftConfig() *Config { + c := &Config{} + err := c.fillDefaults() + if err != nil { + klog.Fatalf("Failed to initialize config: %v", err) + } + return c +} + +func (c *Config) fillDefaults() error { + if c.Debugging.LogLevel == "" { + c.Debugging.LogLevel = "Normal" + } + + if len(c.ApiServer.SubjectAltNames) == 0 { + subjectAltNames, err := getAllHostnames() + if err != nil { + return fmt.Errorf("failed to get all hostnames: %v", err) + } + c.ApiServer.SubjectAltNames = subjectAltNames + } + if c.ApiServer.URL == "" { + c.ApiServer.URL = "https://localhost:6443" + } + + if c.Node.HostnameOverride == "" { + nodeName, err := os.Hostname() + if err != nil { + return fmt.Errorf("Failed to get hostname %v", err) + } + c.Node.HostnameOverride = nodeName + } + if c.Node.NodeIP == "" { + nodeIP, err := util.GetHostIP() + if err != nil { + return fmt.Errorf("failed to get host IP: %v", err) + } + c.Node.NodeIP = nodeIP + } + + if c.DNS.BaseDomain == "" { + c.DNS.BaseDomain = "example.com" + } + + if len(c.Network.ClusterNetwork) == 0 { + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + } + } + if len(c.Network.ServiceNetwork) == 0 { + c.Network.ServiceNetwork = []string{ + "10.43.0.0/16", + } + } + if c.Network.ServiceNodePortRange == "" { + c.Network.ServiceNodePortRange = "30000-32767" + } + if c.Network.DNS == "" { + c.Network.DNS = "10.43.0.10" + } + + if c.Etcd.MinDefragSize == "" { + c.Etcd.MinDefragSize = "100Mi" + } + if c.Etcd.MaxFragmentedPercentage == 0 { + c.Etcd.MaxFragmentedPercentage = 45 + } + if c.Etcd.DefragCheckFreq == "" { + c.Etcd.DefragCheckFreq = "5m" + } + // DoStartupDefrag: true, + if c.Etcd.QuotaBackendSize == "" { + c.Etcd.QuotaBackendSize = "2Gi" + } + + return c.updateComputedValues() +} + +func (c *Config) updateComputedValues() error { + clusterDNS, err := c.computeClusterDNS() + if err != nil { + return err + } + c.Network.DNS = clusterDNS + + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) + if err != nil { + return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) + } + c.Etcd.DefragCheckDuration = d + } + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) + if err != nil { + return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) + } + if !q.IsZero() { + c.Etcd.MinDefragBytes = q.Value() + } + } + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) + if err != nil { + return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) + } + if !q.IsZero() { + c.Etcd.QuotaBackendBytes = q.Value() + } + } + + // If KAS advertise address is not configured then grab it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) + if err != nil { + return fmt.Errorf("error getting apiserver IP: %v", err) + } + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + return nil +} + +func (c *Config) validate() error { + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + + return nil +} + func getAllHostnames() ([]string, error) { cmd := exec.Command("/bin/hostname", "-A") var out bytes.Buffer @@ -45,57 +230,3 @@ func getAllHostnames() ([]string, error) { set := sets.NewString(names...) return set.List(), nil } - -func NewMicroshiftConfig() *Config { - nodeName, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - nodeIP, err := util.GetHostIP() - if err != nil { - klog.Fatalf("failed to get host IP: %v", err) - } - subjectAltNames, err := getAllHostnames() - if err != nil { - klog.Fatalf("failed to get all hostnames: %v", err) - } - - return &Config{ - Debugging: Debugging{ - LogLevel: "Normal", - }, - ApiServer: ApiServer{ - SubjectAltNames: subjectAltNames, - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: nodeName, - NodeIP: nodeIP, - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.42.0.0/16", - }, - }, - ServiceNetwork: []string{ - "10.43.0.0/16", - }, - ServiceNodePortRange: "30000-32767", - DNS: "10.43.0.10", - }, - Etcd: EtcdConfig{ - MinDefragSize: "100Mi", - MinDefragBytes: 100 * 1024 * 1024, // 100MiB - MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: "5m", - DefragCheckDuration: 5 * time.Minute, - DoStartupDefrag: true, - QuotaBackendSize: "2Gi", - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GiB - }, - } -} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go index 1253fe7b46..afeca6fa12 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -3,15 +3,9 @@ package config import ( "errors" "fmt" - "net" - "net/url" "os" - "time" "github.com/mitchellh/go-homedir" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/klog/v2" - ctrl "k8s.io/kubernetes/pkg/controlplane" "sigs.k8s.io/yaml" ) @@ -80,137 +74,48 @@ func findManifestsDir() []string { return manifestsDir } -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) +func parse(contents []byte) (*Config, error) { + c := &Config{} + fmt.Printf("parsing %s\n", string(contents)) + if err := yaml.Unmarshal(contents, c); err != nil { + return nil, fmt.Errorf("Unable to decode configuration: %v", err) } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) + if err := c.fillDefaults(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + if err := c.updateComputedValues(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - c.Etcd = config.Etcd - if c.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) - if err != nil { - return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) - } - c.Etcd.DefragCheckDuration = d + if err := c.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - if c.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) - if err != nil { - return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) - } - if !q.IsZero() { - c.Etcd.MinDefragBytes = q.Value() - } - } - if c.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) - if err != nil { - return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) - } - if !q.IsZero() { - c.Etcd.QuotaBackendBytes = q.Value() - } - } - - return nil + return c, nil } -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then grab it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) - if err != nil { - return fmt.Errorf("error getting apiserver IP: %v", err) - } - c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true +func Read(configFile string) (*Config, error) { + contents, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("reading config file %q: %v", configFile, err) } + return parse(contents) +} - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } +// Get the active configuration. If the configuration file exists, +// read it and require it to be valid. Otherwise return the default +// settings. +func GetActiveConfig() (*Config, error) { + var cfg *Config + filename := GetConfigFile() + _, err := os.Stat(filename) + if os.IsNotExist(err) { + // No configuration file, use the default settings + return NewMicroshiftConfig(), nil + } else if err != nil { + return nil, err } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) + cfg, err = Read(filename) + if err != nil { + return nil, err } - - return nil + return cfg, nil } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go index f7e04e5838..3a3f8cc211 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go @@ -35,17 +35,16 @@ type ClusterNetworkEntry struct { CIDR string `json:"cidr"` } -func (c *Config) computeAndUpdateClusterDNS() error { +func (c *Config) computeClusterDNS() (string, error) { if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") + return "", fmt.Errorf("network.serviceNetwork not filled in") } clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + return "", fmt.Errorf("failed to get DNS IP: %v", err) } - c.Network.DNS = clusterDNS - return nil + return clusterDNS, nil } // getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go index 1622be3ab4..2f88528a29 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go @@ -68,3 +68,10 @@ func (c *Config) validateNodeName(isDefaultNodeName bool) error { return nil } + +func (c *Config) EnsureNodeNameHasNotChanged() error { + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + return c.validateNodeName(c.isDefaultNodeName()) +} diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index b8d72e3405..107f86677d 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -28,12 +28,20 @@ const ( ) func NewRunMicroshiftCommand() *cobra.Command { - cfg := config.NewMicroshiftConfig() - cmd := &cobra.Command{ Use: "run", Short: "Run MicroShift", RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.GetActiveConfig() + if err != nil { + return err + } + // Things to very badly if the node's name has changed + // since the last time the server started. + err = cfg.EnsureNodeNameHasNotChanged() + if err != nil { + return err + } return RunMicroshift(cfg) }, } @@ -42,10 +50,6 @@ func NewRunMicroshiftCommand() *cobra.Command { } func RunMicroshift(cfg *config.Config) error { - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { - klog.Fatalf("Error in reading or validating configuration: %v", err) - } - // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 1ec4bbbd96..2a90932b19 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -3,12 +3,11 @@ package cmd import ( "fmt" + "github.com/openshift/microshift/pkg/config" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" - - "github.com/openshift/microshift/pkg/config" ) type showConfigOptions struct { @@ -21,42 +20,23 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command Mode: "default", } - cfg := config.NewMicroshiftConfig() - cmd := &cobra.Command{ Use: "show-config", Short: "Print MicroShift's configuration", Run: func(cmd *cobra.Command, args []string) { + var cfg *config.Config + var err error - switch opts.Mode { - case "default": - cfg.Node.NodeIP = "" - cfg.Node.HostnameOverride = "" - case "effective": - // Load the current configuration - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { + if opts.Mode == "effective" { + cfg, err = config.GetActiveConfig() + if err != nil { cmdutil.CheckErr(err) } - default: - cmdutil.CheckErr(fmt.Errorf("Unknown mode %q", opts.Mode)) + } else { + cfg = config.NewMicroshiftConfig() } - // map back from internal representation to user config - userCfg := config.Config{ - Network: config.Network{ - ClusterNetwork: []config.ClusterNetworkEntry{ - {CIDR: cfg.Network.ClusterNetwork[0].CIDR}, - }, - ServiceNetwork: []string{cfg.Network.ServiceNetwork[0]}, - ServiceNodePortRange: cfg.Network.ServiceNodePortRange, - }, - DNS: cfg.DNS, - Node: cfg.Node, - ApiServer: cfg.ApiServer, - Debugging: cfg.Debugging, - Etcd: cfg.Etcd, - } - marshalled, err := yaml.Marshal(userCfg) + marshalled, err := yaml.Marshal(cfg) cmdutil.CheckErr(err) fmt.Fprintf(ioStreams.Out, "%s\n", string(marshalled)) diff --git a/pkg/config/config.go b/pkg/config/config.go index 31aaaca3d3..fb7243a797 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,18 +3,23 @@ package config import ( "bytes" "fmt" + "net" + "net/url" "os" "os/exec" "strings" "time" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + ctrl "k8s.io/kubernetes/pkg/controlplane" "github.com/openshift/microshift/pkg/util" ) const ( + // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) @@ -30,6 +35,186 @@ type Config struct { Ingress IngressConfig `json:"-"` } +func NewMicroshiftConfig() *Config { + c := &Config{} + err := c.fillDefaults() + if err != nil { + klog.Fatalf("Failed to initialize config: %v", err) + } + return c +} + +func (c *Config) fillDefaults() error { + if c.Debugging.LogLevel == "" { + c.Debugging.LogLevel = "Normal" + } + + if len(c.ApiServer.SubjectAltNames) == 0 { + subjectAltNames, err := getAllHostnames() + if err != nil { + return fmt.Errorf("failed to get all hostnames: %v", err) + } + c.ApiServer.SubjectAltNames = subjectAltNames + } + if c.ApiServer.URL == "" { + c.ApiServer.URL = "https://localhost:6443" + } + + if c.Node.HostnameOverride == "" { + nodeName, err := os.Hostname() + if err != nil { + return fmt.Errorf("Failed to get hostname %v", err) + } + c.Node.HostnameOverride = nodeName + } + if c.Node.NodeIP == "" { + nodeIP, err := util.GetHostIP() + if err != nil { + return fmt.Errorf("failed to get host IP: %v", err) + } + c.Node.NodeIP = nodeIP + } + + if c.DNS.BaseDomain == "" { + c.DNS.BaseDomain = "example.com" + } + + if len(c.Network.ClusterNetwork) == 0 { + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + } + } + if len(c.Network.ServiceNetwork) == 0 { + c.Network.ServiceNetwork = []string{ + "10.43.0.0/16", + } + } + if c.Network.ServiceNodePortRange == "" { + c.Network.ServiceNodePortRange = "30000-32767" + } + if c.Network.DNS == "" { + c.Network.DNS = "10.43.0.10" + } + + if c.Etcd.MinDefragSize == "" { + c.Etcd.MinDefragSize = "100Mi" + } + if c.Etcd.MaxFragmentedPercentage == 0 { + c.Etcd.MaxFragmentedPercentage = 45 + } + if c.Etcd.DefragCheckFreq == "" { + c.Etcd.DefragCheckFreq = "5m" + } + // DoStartupDefrag: true, + if c.Etcd.QuotaBackendSize == "" { + c.Etcd.QuotaBackendSize = "2Gi" + } + + return c.updateComputedValues() +} + +func (c *Config) updateComputedValues() error { + clusterDNS, err := c.computeClusterDNS() + if err != nil { + return err + } + c.Network.DNS = clusterDNS + + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) + if err != nil { + return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) + } + c.Etcd.DefragCheckDuration = d + } + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) + if err != nil { + return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) + } + if !q.IsZero() { + c.Etcd.MinDefragBytes = q.Value() + } + } + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) + if err != nil { + return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) + } + if !q.IsZero() { + c.Etcd.QuotaBackendBytes = q.Value() + } + } + + // If KAS advertise address is not configured then grab it from the service + // CIDR automatically. + if len(c.ApiServer.AdvertiseAddress) == 0 { + // unchecked error because this was done when getting cluster DNS + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) + _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) + if err != nil { + return fmt.Errorf("error getting apiserver IP: %v", err) + } + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false + } else { + c.ApiServer.SkipInterface = true + } + + return nil +} + +func (c *Config) validate() error { + if len(c.ApiServer.SubjectAltNames) > 0 { + // Any entry in SubjectAltNames will be included in the external access certificates. + // Any of the hostnames and IPs (except the node IP) listed below conflicts with + // other certificates, such as the service network and localhost access. + // The node IP is a bit special. Apiserver k8s service, which holds a service IP + // gets resolved to the node IP. If we include the node IP in the SAN then we have + // an ambiguity, the same IP matches two different certificates and there are errors + // when trying to reach apiserver from within the cluster using the service IP. + // Apiserver will decide which certificate to return to client hello based on SNI + // (which client-go does not use) or raw IP mappings. As soon as there is a match for + // the node IP it returns that certificate, which is the external access one. This + // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate + // is invalid. + u, err := url.Parse(c.ApiServer.URL) + if err != nil { + return fmt.Errorf("failed to parse cluster URL: %v", err) + } + if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { + return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") + } + } else { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { + return fmt.Errorf("subjectAltNames must not contain node IP") + } + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) + } + } + if stringSliceContains( + c.ApiServer.SubjectAltNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + "openshift", + "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", + c.ApiServer.AdvertiseAddress, + ) { + return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") + } + } + + return nil +} + func getAllHostnames() ([]string, error) { cmd := exec.Command("/bin/hostname", "-A") var out bytes.Buffer @@ -45,57 +230,3 @@ func getAllHostnames() ([]string, error) { set := sets.NewString(names...) return set.List(), nil } - -func NewMicroshiftConfig() *Config { - nodeName, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) - } - nodeIP, err := util.GetHostIP() - if err != nil { - klog.Fatalf("failed to get host IP: %v", err) - } - subjectAltNames, err := getAllHostnames() - if err != nil { - klog.Fatalf("failed to get all hostnames: %v", err) - } - - return &Config{ - Debugging: Debugging{ - LogLevel: "Normal", - }, - ApiServer: ApiServer{ - SubjectAltNames: subjectAltNames, - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: nodeName, - NodeIP: nodeIP, - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.42.0.0/16", - }, - }, - ServiceNetwork: []string{ - "10.43.0.0/16", - }, - ServiceNodePortRange: "30000-32767", - DNS: "10.43.0.10", - }, - Etcd: EtcdConfig{ - MinDefragSize: "100Mi", - MinDefragBytes: 100 * 1024 * 1024, // 100MiB - MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: "5m", - DefragCheckDuration: 5 * time.Minute, - DoStartupDefrag: true, - QuotaBackendSize: "2Gi", - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GiB - }, - } -} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a63df0a095..b5b46ba0f9 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -4,9 +4,6 @@ import ( "os" "path/filepath" "testing" - "time" - - "sigs.k8s.io/yaml" "github.com/stretchr/testify/assert" ) @@ -27,102 +24,131 @@ func setupSuiteDataDir(t *testing.T) func() { } } -func TestConfigFile(t *testing.T) { +func TestParse(t *testing.T) { + mkConfig := func() *Config { + c := NewMicroshiftConfig() + c.ApiServer.SkipInterface = true + return c + } + var ttests = []struct { - config Config - expected Config + name string + config string + expected *Config expectErr bool }{ { - config: Config{ - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - }, - Debugging: Debugging{ - LogLevel: "Debug", - }, - Etcd: EtcdConfig{ - QuotaBackendSize: "2Gi", - MinDefragSize: "100Mi", - MaxFragmentedPercentage: 45, - DefragCheckFreq: "5m", - DoStartupDefrag: true, - }, - }, - expected: Config{ - Debugging: Debugging{ - LogLevel: "Debug", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, + name: "empty", + config: "", + expected: mkConfig(), + }, + { + name: "dns", + config: ` +dns: + baseDomain: test-example.com +`, + expected: func() *Config { + c := mkConfig() + c.DNS.BaseDomain = "test-example.com" + return c + }(), + }, + { + name: "network", + config: ` +network: + clusterNetwork: + - cidr: "10.20.30.40/16" + serviceNetwork: + - "40.30.20.10/16" + serviceNodePortRange: "1024-32767" +`, + expected: func() *Config { + c := mkConfig() + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - DNS: "40.30.0.10", - }, - Etcd: EtcdConfig{ - QuotaBackendSize: "2Gi", - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, - MinDefragSize: "100Mi", - MinDefragBytes: 100 * 1024 * 1024, - MaxFragmentedPercentage: 45, - DefragCheckFreq: "5m", - DefragCheckDuration: 5 * time.Minute, - DoStartupDefrag: true, - }, - }, - expectErr: false, + } + c.Network.ServiceNetwork = []string{"40.30.20.10/16"} + c.Network.ServiceNodePortRange = "1024-32767" + c.ApiServer.AdvertiseAddress = "40.30.0.1" + c.updateComputedValues() // recomputes DNS field + return c + }(), + }, + { + name: "node", + config: ` +node: + hostnameOverride: "node1" + nodeIP: "1.2.3.4" +`, + expected: func() *Config { + c := mkConfig() + c.Node.HostnameOverride = "node1" + c.Node.NodeIP = "1.2.3.4" + return c + }(), + }, + { + name: "api-server", + config: ` +apiServer: + subjectAltNames: + - node1 + - node2 +`, + expected: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{ + "node1", "node2", + } + return c + }(), + }, + { + name: "debugging", + config: ` +debugging: + logLevel: Info +`, + expected: func() *Config { + c := mkConfig() + c.Debugging.LogLevel = "Info" + return c + }(), + }, + { + name: "etcd", + config: ` +etcd: + quotaBackendSize: 100Gi + minDefragSize: 1Gi + maxFragmentedPercentage: 99 + defragCheckFreq: 55m + doStartupDefrag: true +`, + expected: func() *Config { + c := mkConfig() + c.Etcd.QuotaBackendSize = "100Gi" + c.Etcd.MinDefragSize = "1Gi" + c.Etcd.MaxFragmentedPercentage = 99 + c.Etcd.DefragCheckFreq = "55m" + c.Etcd.DoStartupDefrag = true + c.updateComputedValues() // parse the time duration + // set some of the other expected values explicitly + c.Etcd.QuotaBackendBytes = 100 * 1024 * 1024 * 1024 + c.Etcd.MinDefragBytes = 1024 * 1024 * 1024 + return c + }(), }, } + for _, tt := range ttests { - t.Run("", func(t *testing.T) { - f, err := os.CreateTemp("", "test") - if err != nil { - t.Errorf("unable to create temp file: %v", err) - } - defer os.Remove(f.Name()) - d, err := yaml.Marshal(&tt.config) - if err != nil { - t.Errorf("unable to marshal configuration: %v", err) - } - _, err = f.Write(d) - if err != nil { - t.Errorf("unable to write to file: %v", err) - } - config := NewMicroshiftConfig() - err = config.ReadFromConfigFile(f.Name()) + t.Run(tt.name, func(t *testing.T) { + config, err := parse([]byte(tt.config)) if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } @@ -130,145 +156,70 @@ func TestConfigFile(t *testing.T) { t.Fatalf("Not expecting error and received: %v", err) } if !tt.expectErr { - assert.Equal(t, tt.expected, *config) + assert.Equal(t, tt.expected, config) } }) } } -// test the MicroshiftConfig.ReadAndValidate function to verify that it configures MicroshiftConfig from -// a configuration file. -func TestMicroshiftConfigReadAndValidate(t *testing.T) { +// Test the validation logic +func TestValidate(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() + mkConfig := func() *Config { + c := NewMicroshiftConfig() + c.ApiServer.SkipInterface = true + return c + } + var ttests = []struct { name string - config Config - expected Config + config *Config expectErr bool }{ { - name: "Config OK full", - config: Config{ - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - }, - Debugging: Debugging{ - LogLevel: "Debug", - }, - Etcd: EtcdConfig{ - QuotaBackendSize: "2Gi", - MinDefragSize: "100Mi", - MaxFragmentedPercentage: 45, - DefragCheckFreq: "5m", - DoStartupDefrag: true, - }, - }, - expected: Config{ - Debugging: Debugging{ - LogLevel: "Debug", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - SkipInterface: true, - URL: "https://localhost:6443", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - DNS: "40.30.0.10", - }, - Etcd: EtcdConfig{ - QuotaBackendSize: "2Gi", - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, - MinDefragSize: "100Mi", - MinDefragBytes: 100 * 1024 * 1024, - MaxFragmentedPercentage: 45, - DefragCheckFreq: "5m", - DefragCheckDuration: 5 * time.Minute, - DoStartupDefrag: true, - }, - }, + name: "defaults-ok", + config: NewMicroshiftConfig(), expectErr: false, }, { - name: "Config NOK with bad SAN localhost", - config: Config{ - ApiServer: ApiServer{ - SubjectAltNames: []string{"127.0.0.1", "localhost"}, - }, - }, - expected: Config{}, + name: "subject-alt-names-with-localhost", + config: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{"localhost"} + return c + }(), expectErr: true, }, { - name: "Config NOK with bad SAN kubernetes service", - config: Config{ - ApiServer: ApiServer{ - SubjectAltNames: []string{"kubernetes"}, - }, - }, - expected: Config{}, + name: "subject-alt-names-with-loopback-ipv4", + config: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{"127.0.0.1"} + return c + }(), + expectErr: true, + }, + { + name: "subject-alt-names-with-kubernetes", + config: func() *Config { + c := mkConfig() + c.ApiServer.SubjectAltNames = []string{"kubernetes"} + return c + }(), expectErr: true, }, } for _, tt := range ttests { t.Run(tt.name, func(t *testing.T) { - f, err := os.CreateTemp("", "test") - if err != nil { - t.Errorf("unable to create temp file: %v", err) - } - defer os.Remove(f.Name()) - d, err := yaml.Marshal(&tt.config) - if err != nil { - t.Errorf("unable to marshal configuration: %v", err) - } - _, err = f.Write(d) - if err != nil { - t.Errorf("unable to write to file: %v", err) - } - config := NewMicroshiftConfig() - err = config.ReadAndValidate(f.Name()) + err := tt.config.validate() if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr { - assert.Equal(t, tt.expected, *config) - } }) } } diff --git a/pkg/config/files.go b/pkg/config/files.go index 1253fe7b46..afeca6fa12 100644 --- a/pkg/config/files.go +++ b/pkg/config/files.go @@ -3,15 +3,9 @@ package config import ( "errors" "fmt" - "net" - "net/url" "os" - "time" "github.com/mitchellh/go-homedir" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/klog/v2" - ctrl "k8s.io/kubernetes/pkg/controlplane" "sigs.k8s.io/yaml" ) @@ -80,137 +74,48 @@ func findManifestsDir() []string { return manifestsDir } -func (c *Config) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) +func parse(contents []byte) (*Config, error) { + c := &Config{} + fmt.Printf("parsing %s\n", string(contents)) + if err := yaml.Unmarshal(contents, c); err != nil { + return nil, fmt.Errorf("Unable to decode configuration: %v", err) } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) + if err := c.fillDefaults(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - // Wire new Config type to existing Config - c.Node = config.Node - c.Debugging = config.Debugging - c.Network = config.Network - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) + if err := c.updateComputedValues(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - - c.DNS = config.DNS - c.ApiServer = config.ApiServer - c.ApiServer.URL = "https://localhost:6443" - - c.Etcd = config.Etcd - if c.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) - if err != nil { - return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) - } - c.Etcd.DefragCheckDuration = d + if err := c.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) } - if c.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) - if err != nil { - return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) - } - if !q.IsZero() { - c.Etcd.MinDefragBytes = q.Value() - } - } - if c.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) - if err != nil { - return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) - } - if !q.IsZero() { - c.Etcd.QuotaBackendBytes = q.Value() - } - } - - return nil + return c, nil } -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *Config) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } - - if err := c.computeAndUpdateClusterDNS(); err != nil { - return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) - } - - // If KAS advertise address is not configured then grab it from the service - // CIDR automatically. - if len(c.ApiServer.AdvertiseAddress) == 0 { - // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) - _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) - if err != nil { - return fmt.Errorf("error getting apiserver IP: %v", err) - } - c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() - c.ApiServer.SkipInterface = false - } else { - c.ApiServer.SkipInterface = true +func Read(configFile string) (*Config, error) { + contents, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("reading config file %q: %v", configFile, err) } + return parse(contents) +} - if len(c.ApiServer.SubjectAltNames) > 0 { - // Any entry in SubjectAltNames will be included in the external access certificates. - // Any of the hostnames and IPs (except the node IP) listed below conflicts with - // other certificates, such as the service network and localhost access. - // The node IP is a bit special. Apiserver k8s service, which holds a service IP - // gets resolved to the node IP. If we include the node IP in the SAN then we have - // an ambiguity, the same IP matches two different certificates and there are errors - // when trying to reach apiserver from within the cluster using the service IP. - // Apiserver will decide which certificate to return to client hello based on SNI - // (which client-go does not use) or raw IP mappings. As soon as there is a match for - // the node IP it returns that certificate, which is the external access one. This - // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate - // is invalid. - u, err := url.Parse(c.ApiServer.URL) - if err != nil { - return fmt.Errorf("failed to parse cluster URL: %v", err) - } - if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { - return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") - } - } else { - if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { - return fmt.Errorf("subjectAltNames must not contain node IP") - } - if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { - return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) - } - } - - if stringSliceContains( - c.ApiServer.SubjectAltNames, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - "openshift", - "openshift.default", - "openshift.default.svc", - "openshift.default.svc.cluster.local", - c.ApiServer.AdvertiseAddress, - ) { - return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") - } +// Get the active configuration. If the configuration file exists, +// read it and require it to be valid. Otherwise return the default +// settings. +func GetActiveConfig() (*Config, error) { + var cfg *Config + filename := GetConfigFile() + _, err := os.Stat(filename) + if os.IsNotExist(err) { + // No configuration file, use the default settings + return NewMicroshiftConfig(), nil + } else if err != nil { + return nil, err } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) + cfg, err = Read(filename) + if err != nil { + return nil, err } - - return nil + return cfg, nil } diff --git a/pkg/config/network.go b/pkg/config/network.go index f7e04e5838..3a3f8cc211 100644 --- a/pkg/config/network.go +++ b/pkg/config/network.go @@ -35,17 +35,16 @@ type ClusterNetworkEntry struct { CIDR string `json:"cidr"` } -func (c *Config) computeAndUpdateClusterDNS() error { +func (c *Config) computeClusterDNS() (string, error) { if len(c.Network.ServiceNetwork) == 0 { - return fmt.Errorf("network.serviceNetwork not filled in") + return "", fmt.Errorf("network.serviceNetwork not filled in") } clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + return "", fmt.Errorf("failed to get DNS IP: %v", err) } - c.Network.DNS = clusterDNS - return nil + return clusterDNS, nil } // getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork diff --git a/pkg/config/node.go b/pkg/config/node.go index 1622be3ab4..2f88528a29 100644 --- a/pkg/config/node.go +++ b/pkg/config/node.go @@ -68,3 +68,10 @@ func (c *Config) validateNodeName(isDefaultNodeName bool) error { return nil } + +func (c *Config) EnsureNodeNameHasNotChanged() error { + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + return c.validateNodeName(c.isDefaultNodeName()) +} From 2d193c3354c02462c6da90d12fbe290f3ce040d2 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Mon, 6 Mar 2023 18:07:19 -0500 Subject: [PATCH 10/10] USHIFT-936: show configuration on startup --- pkg/cmd/run.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 107f86677d..c6f17622ac 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "time" @@ -21,6 +22,7 @@ import ( "github.com/spf13/cobra" "k8s.io/klog/v2" + "sigs.k8s.io/yaml" ) const ( @@ -49,12 +51,25 @@ func NewRunMicroshiftCommand() *cobra.Command { return cmd } +func logConfig(cfg *config.Config) { + marshalled, err := yaml.Marshal(cfg) + if err != nil { + klog.Fatal(err) + } + klog.Info("Effective configuration:") + for _, line := range strings.Split(string(marshalled), "\n") { + klog.Info(line) + } +} + func RunMicroshift(cfg *config.Config) error { // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") } + logConfig(cfg) + // TO-DO: When multi-node is ready, we need to add the controller host-name/mDNS hostname // or VIP to this list on start // see https://github.com/openshift/microshift/pull/471