From 7dbcb35d9c188a37fff10259d608608c380fa34a Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:23:57 -0500 Subject: [PATCH 1/7] USHIFT-936: align the DNS portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 74 +++++++++++++++---- pkg/cmd/init.go | 8 +- pkg/cmd/run.go | 2 +- pkg/cmd/showConfig.go | 4 +- pkg/components/render.go | 2 +- pkg/config/config.go | 11 +-- pkg/config/config_test.go | 8 +- pkg/controllers/kube-apiserver.go | 2 +- 8 files changed, 80 insertions(+), 31 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index cc2c3247fd..53be1d09c6 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/mitchellh/go-homedir" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/component-base/logs" "k8s.io/klog/v2" @@ -58,7 +59,7 @@ type IngressConfig struct { ServingKey []byte } -type EtcdConfig struct { +type InternalEtcdConfig struct { // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value QuotaBackendBytes int64 // If the backend is fragmented more than `maxFragmentedPercentage` @@ -71,6 +72,19 @@ type EtcdConfig struct { DoStartupDefrag bool } +type EtcdConfig struct { + // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value + QuotaBackendSize string + // If the backend is fragmented more than `maxFragmentedPercentage` + // and the database size is greater than `minDefragSize`, do a defrag. + MinDefragSize string + MaxFragmentedPercentage float64 + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). + DefragCheckFreq string + // Whether or not to do a defrag when the server finishes starting + DoStartupDefrag bool +} + type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` @@ -86,20 +100,22 @@ type MicroshiftConfig struct { // Determines if kube-apiserver controller should configure the // KASAdvertiseAddress in the loopback interface. Automatically computed. SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd InternalEtcdConfig `json:"etcd"` + + DNS DNS `json:"-"` } // Top level config file type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Debugging Debugging `json:"debugging"` + Etcd EtcdConfig `json:"etcd"` } type Network struct { @@ -231,14 +247,16 @@ func NewMicroshiftConfig() *MicroshiftConfig { SubjectAltNames: subjectAltNames, NodeName: nodeName, NodeIP: nodeIP, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.42.0.0/16", ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: EtcdConfig{ + Etcd: InternalEtcdConfig{ MinDefragBytes: 100 * 1024 * 1024, // 100MB MaxFragmentedPercentage: 45, // percent DefragCheckFreq: 5 * time.Minute, @@ -392,9 +410,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Network.ServiceNodePortRange != "" { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain - } + c.DNS = config.DNS if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } @@ -402,6 +418,36 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } + if config.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) + if err != nil { + return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) + } + c.Etcd.DefragCheckFreq = d + } + if config.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) + if err != nil { + return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) + } + if !q.IsZero() { + c.Etcd.MinDefragBytes = q.Value() + } + } + if config.Etcd.MaxFragmentedPercentage > 0 { + c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage + } + if config.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) + if err != nil { + return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) + } + if !q.IsZero() { + c.Etcd.QuotaBackendBytes = q.Value() + } + } + c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag + return nil } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 01db0bd4f9..2640023640 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -72,7 +72,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err externalCertNames := []string{ cfg.NodeName, - "api." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, } externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add @@ -229,7 +229,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, Hostnames: []string{ - "*.apps." + cfg.BaseDomain, // wildcard for any additional auto-generated domains + "*.apps." + cfg.DNS.BaseDomain, // wildcard for any additional auto-generated domains }, }, ), @@ -285,8 +285,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - "api." + cfg.BaseDomain, - "api-int." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, + "api-int." + cfg.DNS.BaseDomain, apiServerServiceIP.String(), }, }, diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index a7ccef6628..fc84d457d9 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -62,7 +62,7 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { cfg.Cluster.ServiceCIDR, ".svc", ".cluster.local", - "."+cfg.BaseDomain); err != nil { + "."+cfg.DNS.BaseDomain); err != nil { klog.Fatal(err) } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 21c57f5f08..1adfd5a7b0 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -55,9 +55,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: config.DNS{ - BaseDomain: cfg.BaseDomain, - }, + DNS: cfg.DNS, Node: config.Node{ HostnameOverride: cfg.NodeName, NodeIP: cfg.NodeIP, diff --git a/pkg/components/render.go b/pkg/components/render.go index ce2c26c39d..43c79e6478 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -28,7 +28,7 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar "ClusterCIDR": cfg.Cluster.ClusterCIDR, "ServiceCIDR": cfg.Cluster.ServiceCIDR, "ClusterDNS": cfg.Cluster.DNS, - "BaseDomain": cfg.BaseDomain, + "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { params[k] = v diff --git a/pkg/config/config.go b/pkg/config/config.go index 4855f3fc02..53be1d09c6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -100,11 +100,12 @@ type MicroshiftConfig struct { // Determines if kube-apiserver controller should configure the // KASAdvertiseAddress in the loopback interface. Automatically computed. SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` + + DNS DNS `json:"-"` } // Top level config file @@ -246,7 +247,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { SubjectAltNames: subjectAltNames, NodeName: nodeName, NodeIP: nodeIP, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.42.0.0/16", @@ -407,9 +410,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { if config.Network.ServiceNodePortRange != "" { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain - } + c.DNS = config.DNS if len(config.ApiServer.SubjectAltNames) > 0 { c.SubjectAltNames = config.ApiServer.SubjectAltNames } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1be50403c6..804dcecdd7 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -71,7 +71,9 @@ func TestConfigFile(t *testing.T) { NodeName: "node1", NodeIP: "1.2.3.4", KASAdvertiseAddress: "6.7.8.9", - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.20.30.40/16", @@ -172,7 +174,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { NodeIP: "1.2.3.4", KASAdvertiseAddress: "6.7.8.9", SkipKASInterface: true, - BaseDomain: "example.com", + DNS: DNS{ + BaseDomain: "example.com", + }, Cluster: ClusterConfig{ URL: "https://localhost:6443", ClusterCIDR: "10.20.30.40/16", diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index dc018c9f46..cdf6884c17 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -172,7 +172,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { APIVersion: "route.openshift.io/v1", Kind: "HostAssignmentAdmissionConfig", }, - Domain: "apps." + cfg.BaseDomain, + Domain: "apps." + cfg.DNS.BaseDomain, }, }, }, From 7278de6bebcadbc5718e31b758ca1da30795e05c Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:40:43 -0500 Subject: [PATCH 2/7] USHIFT-936: align the Node portion of the config data structures --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 40 +++++++++---------- pkg/cmd/init.go | 16 ++++---- pkg/cmd/run.go | 4 +- pkg/cmd/showConfig.go | 11 ++--- pkg/components/render.go | 4 +- pkg/config/config.go | 40 +++++++++---------- pkg/config/config_test.go | 32 ++++++++------- pkg/loadbalancerservice/controller.go | 2 +- pkg/mdns/controller.go | 4 +- pkg/node/kubelet.go | 4 +- pkg/sysconfwatch/sysconfwatch_linux.go | 2 +- 12 files changed, 78 insertions(+), 85 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 451403a0ae..c59a915ea1 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -92,8 +92,8 @@ func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { s.etcdCfg.LCUrls = url2379 s.etcdCfg.ListenMetricsUrls = setURL([]string{"localhost"}, "2381") - s.etcdCfg.Name = cfg.NodeName - s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.NodeName, "localhost") + s.etcdCfg.Name = cfg.Node.HostnameOverride + s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.Node.HostnameOverride, "localhost") s.etcdCfg.CipherSuites = tlsCipherSuites s.etcdCfg.ClientTLSInfo.CertFile = cryptomaterial.PeerCertPath(etcdServingCertDir) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 53be1d09c6..8dfd211194 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -89,8 +89,6 @@ type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into // the IP configured in the endpoint slice for kubernetes service. Must be @@ -105,7 +103,8 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` } // Top level config file @@ -245,8 +244,10 @@ func NewMicroshiftConfig() *MicroshiftConfig { return &MicroshiftConfig{ LogVLevel: 2, SubjectAltNames: subjectAltNames, - NodeName: nodeName, - NodeIP: nodeIP, + Node: Node{ + HostnameOverride: nodeName, + NodeIP: nodeIP, + }, DNS: DNS{ BaseDomain: "example.com", }, @@ -272,7 +273,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.NodeName == hostname + return c.Node.HostnameOverride == hostname } // Read or set the NodeName that will be used for this MicroShift instance @@ -282,10 +283,10 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.NodeName, nil + return c.Node.HostnameOverride, nil } else if err != nil { return "", err } @@ -294,8 +295,8 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } establishedNodeName, err := c.establishNodeName() @@ -303,14 +304,14 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.NodeName { + if establishedNodeName != c.Node.HostnameOverride { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) + c.Node.HostnameOverride, establishedNodeName) } else { - c.NodeName = establishedNodeName + c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) } } @@ -395,12 +396,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = config.Node.HostnameOverride - } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP - } + c.Node = config.Node if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -504,10 +500,10 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 2640023640..92ad9e3694 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -71,7 +71,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } externalCertNames := []string{ - cfg.NodeName, + cfg.Node.HostnameOverride, "api." + cfg.DNS.BaseDomain, } externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) @@ -81,8 +81,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.NodeIP { - externalCertNames = append(externalCertNames, cfg.NodeIP) + if cfg.KASAdvertiseAddress != cfg.Node.NodeIP { + externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -172,7 +172,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, // userinfo per https://kubernetes.io/docs/reference/access-authn-authz/node/#overview - UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.NodeName, Groups: []string{"system:nodes"}}, + UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.Node.HostnameOverride, Groups: []string{"system:nodes"}}, }, ).WithServingCertificates( &certchains.ServingCertificateSigningRequestInfo{ @@ -180,7 +180,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err Name: "kubelet-server", ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, - Hostnames: []string{cfg.NodeName}, + Hostnames: []string{cfg.Node.HostnameOverride}, }, ), ), @@ -314,7 +314,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-peer:etcd-client", Groups: []string{"system:etcd-peers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, &certchains.PeerCertificateSigningRequestInfo{ CSRMeta: certchains.CSRMeta{ @@ -322,7 +322,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-server:etcd-client", Groups: []string{"system:etcd-servers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, ), ).WithCABundle( @@ -390,7 +390,7 @@ func initKubeconfigs( } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.NodeName, "localhost") { + for _, name := range append(cfg.SubjectAltNames, cfg.Node.HostnameOverride, "localhost") { u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index fc84d457d9..0a5212c1ea 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -56,8 +56,8 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { // see https://github.com/openshift/microshift/pull/471 if err := util.AddToNoProxyEnv( - cfg.NodeIP, - cfg.NodeName, + cfg.Node.NodeIP, + cfg.Node.HostnameOverride, cfg.Cluster.ClusterCIDR, cfg.Cluster.ServiceCIDR, ".svc", diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 1adfd5a7b0..dd0eb19bbb 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -31,8 +31,8 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command switch opts.Mode { case "default": - cfg.NodeIP = "" - cfg.NodeName = "" + cfg.Node.NodeIP = "" + cfg.Node.HostnameOverride = "" case "effective": // Load the current configuration if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { @@ -55,11 +55,8 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: cfg.DNS, - Node: config.Node{ - HostnameOverride: cfg.NodeName, - NodeIP: cfg.NodeIP, - }, + DNS: cfg.DNS, + Node: cfg.Node, ApiServer: config.ApiServer{ SubjectAltNames: cfg.SubjectAltNames, }, diff --git a/pkg/components/render.go b/pkg/components/render.go index 43c79e6478..d9d4f6d70c 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -23,8 +23,8 @@ var templateFuncs = map[string]interface{}{ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, - "NodeName": cfg.NodeName, - "NodeIP": cfg.NodeIP, + "NodeName": cfg.Node.HostnameOverride, + "NodeIP": cfg.Node.NodeIP, "ClusterCIDR": cfg.Cluster.ClusterCIDR, "ServiceCIDR": cfg.Cluster.ServiceCIDR, "ClusterDNS": cfg.Cluster.DNS, diff --git a/pkg/config/config.go b/pkg/config/config.go index 53be1d09c6..8dfd211194 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -89,8 +89,6 @@ type MicroshiftConfig struct { LogVLevel int `json:"logVLevel"` SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into // the IP configured in the endpoint slice for kubernetes service. Must be @@ -105,7 +103,8 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` } // Top level config file @@ -245,8 +244,10 @@ func NewMicroshiftConfig() *MicroshiftConfig { return &MicroshiftConfig{ LogVLevel: 2, SubjectAltNames: subjectAltNames, - NodeName: nodeName, - NodeIP: nodeIP, + Node: Node{ + HostnameOverride: nodeName, + NodeIP: nodeIP, + }, DNS: DNS{ BaseDomain: "example.com", }, @@ -272,7 +273,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { if err != nil { klog.Fatalf("Failed to get hostname %v", err) } - return c.NodeName == hostname + return c.Node.HostnameOverride == hostname } // Read or set the NodeName that will be used for this MicroShift instance @@ -282,10 +283,10 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { if os.IsNotExist(err) { // ensure that dataDir exists os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { + if err := os.WriteFile(filePath, []byte(c.Node.HostnameOverride), 0444); err != nil { return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) } - return c.NodeName, nil + return c.Node.HostnameOverride, nil } else if err != nil { return "", err } @@ -294,8 +295,8 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { // Validate the NodeName to be used for this MicroShift instances func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } establishedNodeName, err := c.establishNodeName() @@ -303,14 +304,14 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { return fmt.Errorf("failed to establish NodeName: %v", err) } - if establishedNodeName != c.NodeName { + if establishedNodeName != c.Node.HostnameOverride { if !isDefaultNodeName { return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) + c.Node.HostnameOverride, establishedNodeName) } else { - c.NodeName = establishedNodeName + c.Node.HostnameOverride = establishedNodeName klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) + "Please consider using a static NodeName in configuration", c.Node.HostnameOverride) } } @@ -395,12 +396,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = config.Node.HostnameOverride - } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP - } + c.Node = config.Node if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -504,10 +500,10 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 804dcecdd7..711485f21a 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -66,10 +66,12 @@ func TestConfigFile(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", + LogVLevel: 4, + SubjectAltNames: []string{"node1", "node2"}, + Node: Node{ + HostnameOverride: "node1", + NodeIP: "1.2.3.4", + }, KASAdvertiseAddress: "6.7.8.9", DNS: DNS{ BaseDomain: "example.com", @@ -168,10 +170,12 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", + LogVLevel: 4, + SubjectAltNames: []string{"node1", "node2"}, + Node: Node{ + HostnameOverride: "node1", + NodeIP: "1.2.3.4", + }, KASAdvertiseAddress: "6.7.8.9", SkipKASInterface: true, DNS: DNS{ @@ -251,7 +255,7 @@ func TestMicroshiftConfigIsDefaultNodeName(t *testing.T) { t.Errorf("expected default IsDefaultNodeName to be true") } - c.NodeName += "-suffix" + c.Node.HostnameOverride += "-suffix" if c.isDefaultNodeName() { t.Errorf("expected default IsDefaultNodeName to be false") } @@ -262,7 +266,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { defer cleanup() c := NewMicroshiftConfig() - c.NodeName = "node1" + c.Node.HostnameOverride = "node1" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err != nil { t.Errorf("failed to validate node name on first call: %v", err) @@ -271,7 +275,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { nodeNameFile := filepath.Join(dataDir, ".nodename") if data, err := os.ReadFile(nodeNameFile); err != nil { t.Errorf("failed to read node name from file %q: %v", nodeNameFile, err) - } else if string(data) != c.NodeName { + } else if string(data) != c.Node.HostnameOverride { t.Errorf("node name file doesn't match the node name in the saved file: %v", err) } @@ -279,7 +283,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err == nil { t.Errorf("validation should have failed for nodename change: %v", err) } @@ -307,7 +311,7 @@ func TestMicroshiftConfigNodeNameValidationFromDefault(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err != nil { t.Errorf("validation should have failed in this case, it must be a warning in logs: %v", err) } @@ -318,7 +322,7 @@ func TestMicroshiftConfigNodeNameValidationBadName(t *testing.T) { defer cleanup() c := NewMicroshiftConfig() - c.NodeName = "1.2.3.4" + c.Node.HostnameOverride = "1.2.3.4" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err == nil { t.Errorf("failed to validate node name.") diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 56fd397051..2a5d9a0ad3 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -36,7 +36,7 @@ var _ servicemanager.Service = &LoadbalancerServiceController{} func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { return &LoadbalancerServiceController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), } } diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 7ff7e62b9b..3daee405c2 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -25,8 +25,8 @@ type MicroShiftmDNSController struct { func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { return &MicroShiftmDNSController{ - NodeIP: cfg.NodeIP, - NodeName: cfg.NodeName, + NodeIP: cfg.Node.NodeIP, + NodeName: cfg.Node.HostnameOverride, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), hostCount: make(map[string]int), } diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 31a75beb5f..c8e5c487a6 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -75,8 +75,8 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { kubeletFlags.BootstrapKubeconfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.KubeConfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.RuntimeCgroups = "/system.slice/crio.service" - kubeletFlags.HostnameOverride = cfg.NodeName - kubeletFlags.NodeIP = cfg.NodeIP + kubeletFlags.HostnameOverride = cfg.Node.HostnameOverride + kubeletFlags.NodeIP = cfg.Node.NodeIP kubeletFlags.ContainerRuntime = "remote" kubeletFlags.RemoteRuntimeEndpoint = "unix:///var/run/crio/crio.sock" kubeletFlags.NodeLabels["node-role.kubernetes.io/control-plane"] = "" diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index b68d159bd6..0c18891ff7 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -55,7 +55,7 @@ func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchContro } return &SysConfWatchController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, timerFd: fd, } } From 8fa67d76f01db01a8dde0a4567b285ec9f05ae85 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 16:56:12 -0500 Subject: [PATCH 3/7] USHIFT-936: align the Debugging portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 15 ++++++++------- pkg/cmd/showConfig.go | 9 +-------- pkg/config/config.go | 15 ++++++++------- pkg/config/config_test.go | 8 ++++++-- pkg/controllers/kube-apiserver.go | 2 +- pkg/controllers/kube-controller-manager.go | 2 +- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 8dfd211194..18b3009054 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -86,8 +86,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into @@ -103,8 +101,9 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` - Node Node `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` + Debugging Debugging `json:"debugging"` } // Top level config file @@ -242,7 +241,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { } return &MicroshiftConfig{ - LogVLevel: 2, + Debugging: Debugging{ + LogLevel: "Normal", + }, SubjectAltNames: subjectAltNames, Node: Node{ HostnameOverride: nodeName, @@ -395,8 +396,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { } // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() c.Node = config.Node + c.Debugging = config.Debugging if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -559,7 +560,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { +func (c *MicroshiftConfig) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index dd0eb19bbb..ed0bdc32c0 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -5,7 +5,6 @@ import ( "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" @@ -43,10 +42,6 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command } // map back from internal representation to user config - logLevels := []string{"", "", "Normal", "", "Debug", "", "Trace", "", "TraceAll"} - if cfg.LogVLevel < 0 || cfg.LogVLevel >= len(logLevels) { - klog.Fatal("logVLevel out of range [0..%d] %d", len(logLevels)-1, cfg.LogVLevel) - } userCfg := config.Config{ Network: config.Network{ ClusterNetwork: []config.ClusterNetworkEntry{ @@ -60,9 +55,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ApiServer: config.ApiServer{ SubjectAltNames: cfg.SubjectAltNames, }, - Debugging: config.Debugging{ - LogLevel: logLevels[cfg.LogVLevel], - }, + Debugging: cfg.Debugging, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 8dfd211194..18b3009054 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -86,8 +86,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - SubjectAltNames []string `json:"subjectAltNames"` // Kube apiserver advertise address to work around the certificates issue // when requiring external access using the node IP. This will turn into @@ -103,8 +101,9 @@ type MicroshiftConfig struct { Ingress IngressConfig `json:"-"` Etcd InternalEtcdConfig `json:"etcd"` - DNS DNS `json:"-"` - Node Node `json:"-"` + DNS DNS `json:"-"` + Node Node `json:"-"` + Debugging Debugging `json:"debugging"` } // Top level config file @@ -242,7 +241,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { } return &MicroshiftConfig{ - LogVLevel: 2, + Debugging: Debugging{ + LogLevel: "Normal", + }, SubjectAltNames: subjectAltNames, Node: Node{ HostnameOverride: nodeName, @@ -395,8 +396,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { } // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() c.Node = config.Node + c.Debugging = config.Debugging if len(config.Network.ClusterNetwork) != 0 { c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR } @@ -559,7 +560,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { +func (c *MicroshiftConfig) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 711485f21a..42ae480122 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -66,7 +66,9 @@ func TestConfigFile(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, + Debugging: Debugging{ + LogLevel: "Debug", + }, SubjectAltNames: []string{"node1", "node2"}, Node: Node{ HostnameOverride: "node1", @@ -170,7 +172,9 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { }, }, expected: MicroshiftConfig{ - LogVLevel: 4, + Debugging: Debugging{ + LogLevel: "Debug", + }, SubjectAltNames: []string{"node1", "node2"}, Node: Node{ HostnameOverride: "node1", diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index cdf6884c17..12babc785d 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -92,7 +92,7 @@ func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { - s.verbosity = cfg.LogVLevel + s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) kubeCSRSignerDir := cryptomaterial.CSRSignerCertDir(certsDir) diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 9c8a30b018..3c15528282 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -93,7 +93,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "use-service-account-credentials": {"true"}, "cluster-signing-cert-file": {clusterSigningCert}, "cluster-signing-key-file": {clusterSigningKey}, - "v": {strconv.Itoa(cfg.LogVLevel)}, + "v": {strconv.Itoa(cfg.GetVerbosity())}, }, } From 92a96adda2738784dc6bf68c8d566bb7a27c0888 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 17:11:24 -0500 Subject: [PATCH 4/7] USHIFT-936: align the Etcd portion of the config data structures --- etcd/cmd/microshift-etcd/run.go | 2 +- .../openshift/microshift/pkg/config/config.go | 63 +++++++++---------- pkg/cmd/showConfig.go | 1 + pkg/config/config.go | 63 +++++++++---------- pkg/config/config_test.go | 14 +++-- 5 files changed, 68 insertions(+), 75 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index c59a915ea1..d0f04bf33e 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -66,7 +66,7 @@ func (s *EtcdService) Name() string { return "etcd" } func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage - s.defragCheckFreq = cfg.Etcd.DefragCheckFreq + s.defragCheckFreq = cfg.Etcd.DefragCheckDuration s.doStartupDefrag = cfg.Etcd.DoStartupDefrag microshiftDataDir := config.GetDataDir() diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 18b3009054..945596b9b8 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -59,30 +59,23 @@ type IngressConfig struct { ServingKey []byte } -type InternalEtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - type EtcdConfig struct { // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string + QuotaBackendSize string `json:"quotaBackendSize"` + QuotaBackendBytes int64 `json:"-"` + // If the backend is fragmented more than `maxFragmentedPercentage` // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string - MaxFragmentedPercentage float64 + MinDefragSize string `json:"minDefragSize"` + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string + DefragCheckFreq string `json:"defragCheckFreq"` + DefragCheckDuration time.Duration `json:"-"` + // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool + DoStartupDefrag bool `json:"doStartupDefrag"` } type MicroshiftConfig struct { @@ -98,8 +91,8 @@ type MicroshiftConfig struct { SkipKASInterface bool `json:"-"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd InternalEtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd EtcdConfig `json:"etcd"` DNS DNS `json:"-"` Node Node `json:"-"` @@ -258,12 +251,15 @@ func NewMicroshiftConfig() *MicroshiftConfig { ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: InternalEtcdConfig{ - MinDefragBytes: 100 * 1024 * 1024, // 100MB + Etcd: EtcdConfig{ + MinDefragSize: "100Mi", + MinDefragBytes: 100 * 1024 * 1024, // 100MiB MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GB + QuotaBackendSize: "2Gi", + QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GiB }, } } @@ -415,15 +411,16 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } - if config.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) + c.Etcd = config.Etcd + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) if err != nil { return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) } - c.Etcd.DefragCheckFreq = d + c.Etcd.DefragCheckDuration = d } - if config.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) if err != nil { return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) } @@ -431,11 +428,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.MinDefragBytes = q.Value() } } - if config.Etcd.MaxFragmentedPercentage > 0 { - c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage - } - if config.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) if err != nil { return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) } @@ -443,7 +437,6 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.QuotaBackendBytes = q.Value() } } - c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag return nil } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index ed0bdc32c0..b88d0b661c 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -56,6 +56,7 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command SubjectAltNames: cfg.SubjectAltNames, }, Debugging: cfg.Debugging, + Etcd: cfg.Etcd, } marshalled, err := yaml.Marshal(userCfg) cmdutil.CheckErr(err) diff --git a/pkg/config/config.go b/pkg/config/config.go index 18b3009054..945596b9b8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -59,30 +59,23 @@ type IngressConfig struct { ServingKey []byte } -type InternalEtcdConfig struct { - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - type EtcdConfig struct { // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendSize string + QuotaBackendSize string `json:"quotaBackendSize"` + QuotaBackendBytes int64 `json:"-"` + // If the backend is fragmented more than `maxFragmentedPercentage` // and the database size is greater than `minDefragSize`, do a defrag. - MinDefragSize string - MaxFragmentedPercentage float64 + MinDefragSize string `json:"minDefragSize"` + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"maxFragmentedPercentage"` + // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq string + DefragCheckFreq string `json:"defragCheckFreq"` + DefragCheckDuration time.Duration `json:"-"` + // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool + DoStartupDefrag bool `json:"doStartupDefrag"` } type MicroshiftConfig struct { @@ -98,8 +91,8 @@ type MicroshiftConfig struct { SkipKASInterface bool `json:"-"` Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` - Etcd InternalEtcdConfig `json:"etcd"` + Ingress IngressConfig `json:"-"` + Etcd EtcdConfig `json:"etcd"` DNS DNS `json:"-"` Node Node `json:"-"` @@ -258,12 +251,15 @@ func NewMicroshiftConfig() *MicroshiftConfig { ServiceCIDR: "10.43.0.0/16", ServiceNodePortRange: "30000-32767", }, - Etcd: InternalEtcdConfig{ - MinDefragBytes: 100 * 1024 * 1024, // 100MB + Etcd: EtcdConfig{ + MinDefragSize: "100Mi", + MinDefragBytes: 100 * 1024 * 1024, // 100MiB MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, - QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GB + QuotaBackendSize: "2Gi", + QuotaBackendBytes: 2 * 1024 * 1024 * 1024, // 2GiB }, } } @@ -415,15 +411,16 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress } - if config.Etcd.DefragCheckFreq != "" { - d, err := time.ParseDuration(config.Etcd.DefragCheckFreq) + c.Etcd = config.Etcd + if c.Etcd.DefragCheckFreq != "" { + d, err := time.ParseDuration(c.Etcd.DefragCheckFreq) if err != nil { return fmt.Errorf("failed to parse etcd defragCheckFreq: %v", err) } - c.Etcd.DefragCheckFreq = d + c.Etcd.DefragCheckDuration = d } - if config.Etcd.MinDefragSize != "" { - q, err := resource.ParseQuantity(config.Etcd.MinDefragSize) + if c.Etcd.MinDefragSize != "" { + q, err := resource.ParseQuantity(c.Etcd.MinDefragSize) if err != nil { return fmt.Errorf("failed to parse etcd minDefragSize: %v", err) } @@ -431,11 +428,8 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.MinDefragBytes = q.Value() } } - if config.Etcd.MaxFragmentedPercentage > 0 { - c.Etcd.MaxFragmentedPercentage = config.Etcd.MaxFragmentedPercentage - } - if config.Etcd.QuotaBackendSize != "" { - q, err := resource.ParseQuantity(config.Etcd.QuotaBackendSize) + if c.Etcd.QuotaBackendSize != "" { + q, err := resource.ParseQuantity(c.Etcd.QuotaBackendSize) if err != nil { return fmt.Errorf("failed to parse etcd quotaBackendSize: %v", err) } @@ -443,7 +437,6 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Etcd.QuotaBackendBytes = q.Value() } } - c.Etcd.DoStartupDefrag = config.Etcd.DoStartupDefrag return nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 42ae480122..12be8b7798 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -84,11 +84,14 @@ func TestConfigFile(t *testing.T) { ServiceCIDR: "40.30.20.10/16", ServiceNodePortRange: "1024-32767", }, - Etcd: InternalEtcdConfig{ + Etcd: EtcdConfig{ + QuotaBackendSize: "2Gi", QuotaBackendBytes: 2 * 1024 * 1024 * 1024, + MinDefragSize: "100Mi", MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, }, }, @@ -192,11 +195,14 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { ServiceNodePortRange: "1024-32767", DNS: "40.30.0.10", }, - Etcd: InternalEtcdConfig{ + Etcd: EtcdConfig{ + QuotaBackendSize: "2Gi", QuotaBackendBytes: 2 * 1024 * 1024 * 1024, + MinDefragSize: "100Mi", MinDefragBytes: 100 * 1024 * 1024, MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, + DefragCheckFreq: "5m", + DefragCheckDuration: 5 * time.Minute, DoStartupDefrag: true, }, }, From 5c77095a226b4d44811419454a019dcc5a3d6fe4 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 17:24:05 -0500 Subject: [PATCH 5/7] USHIFT-936: align the ApiServer portion of the config data structures --- .../openshift/microshift/pkg/config/config.go | 54 +++++++++---------- pkg/cmd/init.go | 6 +-- pkg/cmd/showConfig.go | 8 ++- pkg/config/config.go | 54 +++++++++---------- pkg/config/config_test.go | 14 +++-- pkg/controllers/kube-apiserver.go | 2 +- pkg/node/netconfig.go | 4 +- 7 files changed, 66 insertions(+), 76 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 945596b9b8..c525d35eae 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -79,17 +79,7 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - Cluster ClusterConfig `json:"cluster"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -97,6 +87,7 @@ type MicroshiftConfig struct { DNS DNS `json:"-"` Node Node `json:"-"` Debugging Debugging `json:"debugging"` + ApiServer ApiServer `json:"-"` } // Top level config file @@ -150,9 +141,15 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` } type Node struct { @@ -237,7 +234,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { Debugging: Debugging{ LogLevel: "Normal", }, - SubjectAltNames: subjectAltNames, + ApiServer: ApiServer{ + SubjectAltNames: subjectAltNames, + }, Node: Node{ HostnameOverride: nodeName, NodeIP: nodeIP, @@ -404,12 +403,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } c.DNS = config.DNS - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames - } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress - } + c.ApiServer = config.ApiServer c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -459,20 +453,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // If KAS advertise address is not configured then grab it from the service // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) } - c.KASAdvertiseAddress = apiServerServiceIP.String() - c.SkipKASInterface = false + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false } else { - c.SkipKASInterface = true + c.ApiServer.SkipInterface = true } - if len(c.SubjectAltNames) > 0 { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -490,20 +484,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -512,7 +506,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 92ad9e3694..79faf8bb9f 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -74,14 +74,14 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err cfg.Node.HostnameOverride, "api." + cfg.DNS.BaseDomain, } - externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) + externalCertNames = append(externalCertNames, cfg.ApiServer.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add // it to the certificates or else the internal pod access to apiserver is // broken. Because of client-go not using SNI and the way apiserver handles // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.Node.NodeIP { + if cfg.ApiServer.AdvertiseAddress != cfg.Node.NodeIP { externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } @@ -390,7 +390,7 @@ func initKubeconfigs( } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.Node.HostnameOverride, "localhost") { + for _, name := range append(cfg.ApiServer.SubjectAltNames, cfg.Node.HostnameOverride, "localhost") { u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index b88d0b661c..95848e3de3 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -50,11 +50,9 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, }, - DNS: cfg.DNS, - Node: cfg.Node, - ApiServer: config.ApiServer{ - SubjectAltNames: cfg.SubjectAltNames, - }, + DNS: cfg.DNS, + Node: cfg.Node, + ApiServer: cfg.ApiServer, Debugging: cfg.Debugging, Etcd: cfg.Etcd, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 945596b9b8..c525d35eae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -79,17 +79,7 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - SubjectAltNames []string `json:"subjectAltNames"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - Cluster ClusterConfig `json:"cluster"` + Cluster ClusterConfig `json:"cluster"` Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -97,6 +87,7 @@ type MicroshiftConfig struct { DNS DNS `json:"-"` Node Node `json:"-"` Debugging Debugging `json:"debugging"` + ApiServer ApiServer `json:"-"` } // Top level config file @@ -150,9 +141,15 @@ type DNS struct { type ApiServer struct { // SubjectAltNames added to API server certs SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` } type Node struct { @@ -237,7 +234,9 @@ func NewMicroshiftConfig() *MicroshiftConfig { Debugging: Debugging{ LogLevel: "Normal", }, - SubjectAltNames: subjectAltNames, + ApiServer: ApiServer{ + SubjectAltNames: subjectAltNames, + }, Node: Node{ HostnameOverride: nodeName, NodeIP: nodeIP, @@ -404,12 +403,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange } c.DNS = config.DNS - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames - } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress - } + c.ApiServer = config.ApiServer c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -459,20 +453,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // If KAS advertise address is not configured then grab it from the service // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) } - c.KASAdvertiseAddress = apiServerServiceIP.String() - c.SkipKASInterface = false + c.ApiServer.AdvertiseAddress = apiServerServiceIP.String() + c.ApiServer.SkipInterface = false } else { - c.SkipKASInterface = true + c.ApiServer.SkipInterface = true } - if len(c.SubjectAltNames) > 0 { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -490,20 +484,20 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.Node.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -512,7 +506,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 12be8b7798..0e6cf09314 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -69,12 +69,14 @@ func TestConfigFile(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - SubjectAltNames: []string{"node1", "node2"}, + ApiServer: ApiServer{ + SubjectAltNames: []string{"node1", "node2"}, + AdvertiseAddress: "6.7.8.9", + }, Node: Node{ HostnameOverride: "node1", NodeIP: "1.2.3.4", }, - KASAdvertiseAddress: "6.7.8.9", DNS: DNS{ BaseDomain: "example.com", }, @@ -178,13 +180,15 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { Debugging: Debugging{ LogLevel: "Debug", }, - SubjectAltNames: []string{"node1", "node2"}, + ApiServer: ApiServer{ + SubjectAltNames: []string{"node1", "node2"}, + AdvertiseAddress: "6.7.8.9", + SkipInterface: true, + }, Node: Node{ HostnameOverride: "node1", NodeIP: "1.2.3.4", }, - KASAdvertiseAddress: "6.7.8.9", - SkipKASInterface: true, DNS: DNS{ BaseDomain: "example.com", }, diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 12babc785d..c627c5f7ce 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -117,7 +117,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { s.masterURL = cfg.Cluster.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) - s.advertiseAddress = cfg.KASAdvertiseAddress + s.advertiseAddress = cfg.ApiServer.AdvertiseAddress overrides := &kubecontrolplanev1.KubeAPIServerConfig{ APIServerArguments: map[string]kubecontrolplanev1.Arguments{ diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e43813d7f8..e92f9c81d3 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -48,8 +48,8 @@ func (n *NetworkConfiguration) Name() string { return componentNetwork func (n *NetworkConfiguration) Dependencies() []string { return []string{} } func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { - n.kasAdvertiseAddress = cfg.KASAdvertiseAddress - n.skipInterfaceConfiguration = cfg.SkipKASInterface + n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress + n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { From a7b8f86271ff18f3fe3681b455b0b3f912cf2889 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 18:23:53 -0500 Subject: [PATCH 6/7] USHIFT-936: align the Cluster/Network portions of the config data structures Also introduces assert package in the test code for easier diagnosis of unit test failures. --- .../openshift/microshift/pkg/config/config.go | 67 ++++++++++------- pkg/cmd/init.go | 18 ++--- pkg/cmd/run.go | 4 +- pkg/cmd/showConfig.go | 6 +- pkg/components/controllers.go | 2 +- pkg/components/render.go | 6 +- pkg/config/config.go | 73 +++++++++++-------- pkg/config/config_test.go | 36 +++++---- pkg/controllers/kube-apiserver.go | 10 +-- pkg/controllers/kube-controller-manager.go | 2 +- pkg/node/kubelet.go | 2 +- 11 files changed, 132 insertions(+), 94 deletions(-) diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index c525d35eae..5d1a2c7a05 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -47,11 +47,9 @@ var ( ) type ClusterConfig struct { - URL string `json:"-"` ClusterCIDR string `json:"clusterCIDR"` ServiceCIDR string `json:"serviceCIDR"` ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` } type IngressConfig struct { @@ -79,8 +77,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -88,6 +84,7 @@ type MicroshiftConfig struct { Node Node `json:"-"` Debugging Debugging `json:"debugging"` ApiServer ApiServer `json:"-"` + Network Network `json:"-"` } // Top level config file @@ -118,6 +115,9 @@ type Network struct { // installed. // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` + + // The DNS server to use + DNS string `json:"-"` } type ClusterNetworkEntry struct { @@ -150,6 +150,9 @@ type ApiServer struct { // Determines if kube-apiserver controller should configure the // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` } type Node struct { @@ -236,6 +239,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { }, ApiServer: ApiServer{ SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: nodeName, @@ -244,11 +248,17 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", }, Etcd: EtcdConfig{ MinDefragSize: "100Mi", @@ -315,10 +325,10 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { +func (c *MicroshiftConfig) ApiServerPort() (int, error) { var port string - parsed, err := url.Parse(c.URL) + parsed, err := url.Parse(c.ApiServer.URL) if err != nil { return 0, err } @@ -393,17 +403,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.Node = config.Node c.Debugging = config.Debugging - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } + c.DNS = config.DNS c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -435,6 +442,19 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } +func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + // Note: add a configFile parameter here because of unit test requiring custom // local directory func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { @@ -444,18 +464,15 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } } - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } - c.Cluster.DNS = clusterDNS // If KAS advertise address is not configured then grab it from the service // CIDR automatically. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) @@ -479,7 +496,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index 79faf8bb9f..d5faea3f98 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -60,7 +60,7 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { - _, svcNet, err := net.ParseCIDR(cfg.Cluster.ServiceCIDR) + _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err } @@ -380,11 +380,11 @@ func initKubeconfigs( return err } - u, err := url.Parse(cfg.Cluster.URL) + u, err := url.Parse(cfg.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } - apiServerPort, err := cfg.Cluster.ApiServerPort() + apiServerPort, err := cfg.ApiServerPort() if err != nil { return fmt.Errorf("failed to get apiserver port: %v", err) } @@ -405,7 +405,7 @@ func initKubeconfigs( if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeAdmin), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, adminKubeconfigCertPEM, adminKubeconfigKeyPEM, @@ -419,7 +419,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, kcmCertPEM, kcmKeyPEM, @@ -433,7 +433,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeScheduler), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, schedulerCertPEM, schedulerKeyPEM, ); err != nil { @@ -446,7 +446,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.Kubelet), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, kubeletCertPEM, kubeletKeyPEM, ); err != nil { @@ -458,7 +458,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.ClusterPolicyController), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, clusterPolicyControllerCertPEM, clusterPolicyControllerKeyPEM, ); err != nil { @@ -471,7 +471,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.RouteControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, inClusterTrustBundlePEM, routeControllerManagerCertPEM, routeControllerManagerKeyPEM, ); err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 0a5212c1ea..cf9294a644 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -58,8 +58,8 @@ func RunMicroshift(cfg *config.MicroshiftConfig) error { if err := util.AddToNoProxyEnv( cfg.Node.NodeIP, cfg.Node.HostnameOverride, - cfg.Cluster.ClusterCIDR, - cfg.Cluster.ServiceCIDR, + cfg.Network.ClusterNetwork[0].CIDR, + cfg.Network.ServiceNetwork[0], ".svc", ".cluster.local", "."+cfg.DNS.BaseDomain); err != nil { diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 95848e3de3..1ec4bbbd96 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -45,10 +45,10 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command userCfg := config.Config{ Network: config.Network{ ClusterNetwork: []config.ClusterNetworkEntry{ - {CIDR: cfg.Cluster.ClusterCIDR}, + {CIDR: cfg.Network.ClusterNetwork[0].CIDR}, }, - ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, - ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServiceNetwork: []string{cfg.Network.ServiceNetwork[0]}, + ServiceNodePortRange: cfg.Network.ServiceNodePortRange, }, DNS: cfg.DNS, Node: cfg.Node, diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index 10ae6593c1..b69f965cf7 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -210,7 +210,7 @@ func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) err } extraParams := assets.RenderParams{ - "ClusterIP": cfg.Cluster.DNS, + "ClusterIP": cfg.Network.DNS, } if err := assets.ApplyServices(svc, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply service %v %v", svc, err) diff --git a/pkg/components/render.go b/pkg/components/render.go index d9d4f6d70c..b545d920de 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -25,9 +25,9 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar "ReleaseImage": release.Image, "NodeName": cfg.Node.HostnameOverride, "NodeIP": cfg.Node.NodeIP, - "ClusterCIDR": cfg.Cluster.ClusterCIDR, - "ServiceCIDR": cfg.Cluster.ServiceCIDR, - "ClusterDNS": cfg.Cluster.DNS, + "ClusterCIDR": cfg.Network.ClusterNetwork[0].CIDR, + "ServiceCIDR": cfg.Network.ServiceNetwork[0], + "ClusterDNS": cfg.Network.DNS, "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { diff --git a/pkg/config/config.go b/pkg/config/config.go index c525d35eae..89bc81cb58 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -46,14 +46,6 @@ var ( manifestsDir = findManifestsDir() ) -type ClusterConfig struct { - URL string `json:"-"` - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` -} - type IngressConfig struct { ServingCertificate []byte ServingKey []byte @@ -79,8 +71,6 @@ type EtcdConfig struct { } type MicroshiftConfig struct { - Cluster ClusterConfig `json:"cluster"` - Ingress IngressConfig `json:"-"` Etcd EtcdConfig `json:"etcd"` @@ -88,6 +78,7 @@ type MicroshiftConfig struct { Node Node `json:"-"` Debugging Debugging `json:"debugging"` ApiServer ApiServer `json:"-"` + Network Network `json:"-"` } // Top level config file @@ -118,6 +109,9 @@ type Network struct { // installed. // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` + + // The DNS server to use + DNS string `json:"-"` } type ClusterNetworkEntry struct { @@ -150,6 +144,9 @@ type ApiServer struct { // Determines if kube-apiserver controller should configure the // AdvertiseAddress in the loopback interface. Automatically computed. SkipInterface bool `json:"-"` + + // The URL of the API server + URL string `json:"-"` } type Node struct { @@ -236,6 +233,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { }, ApiServer: ApiServer{ SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: nodeName, @@ -244,11 +242,17 @@ func NewMicroshiftConfig() *MicroshiftConfig { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", }, Etcd: EtcdConfig{ MinDefragSize: "100Mi", @@ -315,10 +319,10 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { +func (c *MicroshiftConfig) ApiServerPort() (int, error) { var port string - parsed, err := url.Parse(c.URL) + parsed, err := url.Parse(c.ApiServer.URL) if err != nil { return 0, err } @@ -393,17 +397,14 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { // Wire new Config type to existing MicroshiftConfig c.Node = config.Node c.Debugging = config.Debugging - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange + c.Network = config.Network + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } + c.DNS = config.DNS c.ApiServer = config.ApiServer + c.ApiServer.URL = "https://localhost:6443" c.Etcd = config.Etcd if c.Etcd.DefragCheckFreq != "" { @@ -435,6 +436,19 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } +func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { + if len(c.Network.ServiceNetwork) == 0 { + return fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return fmt.Errorf("failed to get DNS IP: %v", err) + } + c.Network.DNS = clusterDNS + return nil +} + // Note: add a configFile parameter here because of unit test requiring custom // local directory func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { @@ -444,18 +458,15 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } } - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) - if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + if err := c.computeAndUpdateClusterDNS(); err != nil { + return fmt.Errorf("Failed to validate configuration file %s: %v", configFile, err) } - c.Cluster.DNS = clusterDNS // If KAS advertise address is not configured then grab it from the service // CIDR automatically. if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) _, apiServerServiceIP, err := ctrl.ServiceIPRange(*svcNet) if err != nil { return fmt.Errorf("error getting apiserver IP: %v", err) @@ -479,7 +490,7 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 0e6cf09314..999ad3bc4b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -3,11 +3,12 @@ package config import ( "os" "path/filepath" - "reflect" "testing" "time" "sigs.k8s.io/yaml" + + "github.com/stretchr/testify/assert" ) const ( @@ -72,6 +73,7 @@ func TestConfigFile(t *testing.T) { ApiServer: ApiServer{ SubjectAltNames: []string{"node1", "node2"}, AdvertiseAddress: "6.7.8.9", + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: "node1", @@ -80,11 +82,15 @@ func TestConfigFile(t *testing.T) { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", + }, + }, + ServiceNetwork: []string{"40.30.20.10/16"}, ServiceNodePortRange: "1024-32767", + DNS: "40.30.0.10", }, Etcd: EtcdConfig{ QuotaBackendSize: "2Gi", @@ -123,8 +129,8 @@ func TestConfigFile(t *testing.T) { if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadFromConfigFile() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + assert.Equal(t, tt.expected, *config) } }) } @@ -184,6 +190,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"node1", "node2"}, AdvertiseAddress: "6.7.8.9", SkipInterface: true, + URL: "https://localhost:6443", }, Node: Node{ HostnameOverride: "node1", @@ -192,10 +199,13 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { DNS: DNS{ BaseDomain: "example.com", }, - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", + Network: Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", + }, + }, + ServiceNetwork: []string{"40.30.20.10/16"}, ServiceNodePortRange: "1024-32767", DNS: "40.30.0.10", }, @@ -256,8 +266,8 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadAndValidate() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + assert.Equal(t, tt.expected, *config) } }) } diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index c627c5f7ce..16e9db4f61 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -110,12 +110,12 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { } // Get the apiserver port so we can set it as an argument - apiServerPort, err := cfg.Cluster.ApiServerPort() + apiServerPort, err := cfg.ApiServerPort() if err != nil { return err } - s.masterURL = cfg.Cluster.URL + s.masterURL = cfg.ApiServer.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) s.advertiseAddress = cfg.ApiServer.AdvertiseAddress @@ -139,7 +139,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { "proxy-client-key-file": {cryptomaterial.ClientKeyPath(aggregatorClientCertDir)}, "requestheader-client-ca-file": {aggregatorCAPath}, "service-account-signing-key-file": {microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key"}, - "service-node-port-range": {cfg.Cluster.ServiceNodePortRange}, + "service-node-port-range": {cfg.Network.ServiceNodePortRange}, "tls-cert-file": {servingCert}, "tls-private-key-file": {servingKey}, "disable-admission-plugins": { @@ -214,8 +214,8 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { ServiceAccountPublicKeyFiles: []string{ microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.pub", }, - ServicesSubnet: cfg.Cluster.ServiceCIDR, - ServicesNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServicesSubnet: cfg.Network.ServiceNetwork[0], + ServicesNodePortRange: cfg.Network.ServiceNodePortRange, } overridesBytes, err := json.Marshal(overrides) diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 3c15528282..70ebdff2ac 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -85,7 +85,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "authorization-kubeconfig": {kubeConfig}, "service-account-private-key-file": {kcmServiceAccountPrivateKeyFile()}, "allocate-node-cidrs": {"true"}, - "cluster-cidr": {cfg.Cluster.ClusterCIDR}, + "cluster-cidr": {cfg.Network.ClusterNetwork[0].CIDR}, "root-ca-file": {kcmRootCAFile()}, "bind-address": {"127.0.0.1"}, "secure-port": {"10257"}, diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index c8e5c487a6..ebc246ded8 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -112,7 +112,7 @@ cgroupDriver: "systemd" failSwapOn: false volumePluginDir: ` + microshiftDataDir + `/kubelet-plugins/volume/exec clusterDNS: - - ` + cfg.Cluster.DNS + ` + - ` + cfg.Network.DNS + ` clusterDomain: cluster.local containerLogMaxSize: 50Mi maxPods: 250 From c0b1b5db77c4339734ab63a2fe7dedb04dcb04e6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 3 Mar 2023 18:36:13 -0500 Subject: [PATCH 7/7] USHIFT-936: merge MicroshiftConfig into Config --- etcd/cmd/microshift-etcd/run.go | 4 +- .../openshift/microshift/pkg/config/config.go | 43 ++++++++----------- pkg/assets/crd.go | 4 +- pkg/cmd/init.go | 6 +-- pkg/cmd/run.go | 2 +- pkg/components/components.go | 2 +- pkg/components/controllers.go | 6 +-- pkg/components/networking.go | 2 +- pkg/components/render.go | 2 +- pkg/components/storage.go | 2 +- pkg/config/config.go | 43 ++++++++----------- pkg/config/config_test.go | 12 +++--- pkg/controllers/cluster-policy-controller.go | 4 +- pkg/controllers/etcd.go | 2 +- pkg/controllers/infra-services-controller.go | 6 +-- pkg/controllers/kube-apiserver.go | 6 +-- pkg/controllers/kube-controller-manager.go | 4 +- pkg/controllers/kube-scheduler.go | 6 +-- pkg/controllers/openshift-crd-manager.go | 4 +- .../openshift-default-scc-manager.go | 6 +-- .../openshift-route-controller-manager.go | 6 +-- pkg/controllers/version.go | 4 +- pkg/kustomize/apply.go | 2 +- pkg/loadbalancerservice/controller.go | 2 +- pkg/mdns/controller.go | 2 +- pkg/node/kubelet.go | 6 +-- pkg/node/netconfig.go | 4 +- pkg/sysconfwatch/sysconfwatch_linux.go | 2 +- 28 files changed, 88 insertions(+), 106 deletions(-) diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index d0f04bf33e..67f3cb9a41 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -55,7 +55,7 @@ type EtcdService struct { doStartupDefrag bool } -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { s := &EtcdService{} s.configure(cfg) return s @@ -63,7 +63,7 @@ func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { func (s *EtcdService) Name() string { return "etcd" } -func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { +func (s *EtcdService) configure(cfg *config.Config) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage s.defragCheckFreq = cfg.Etcd.DefragCheckDuration diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 5d1a2c7a05..bc8437a1e9 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -76,25 +76,16 @@ type EtcdConfig struct { DoStartupDefrag bool `json:"doStartupDefrag"` } -type MicroshiftConfig struct { - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` - - DNS DNS `json:"-"` - Node Node `json:"-"` - Debugging Debugging `json:"debugging"` - ApiServer ApiServer `json:"-"` - Network Network `json:"-"` -} - -// Top level config file type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` Node Node `json:"node"` ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` + + // Internal-only fields + Ingress IngressConfig `json:"-"` } type Network struct { @@ -195,11 +186,11 @@ const ( ) // KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { return filepath.Join(dataDir, "resources", string(id), "kubeconfig") } -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { +func (cfg *Config) KubeConfigAdminPath(id string) string { return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") } @@ -219,7 +210,7 @@ func getAllHostnames() ([]string, error) { return set.List(), nil } -func NewMicroshiftConfig() *MicroshiftConfig { +func NewMicroshiftConfig() *Config { nodeName, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -233,7 +224,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { klog.Fatalf("failed to get all hostnames: %v", err) } - return &MicroshiftConfig{ + return &Config{ Debugging: Debugging{ LogLevel: "Normal", }, @@ -274,7 +265,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { } // Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { +func (c *Config) isDefaultNodeName() bool { hostname, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -283,7 +274,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { } // Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { +func (c *Config) establishNodeName() (string, error) { filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { @@ -300,7 +291,7 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { } // Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { +func (c *Config) validateNodeName(isDefaultNodeName bool) error { if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } @@ -325,7 +316,7 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *MicroshiftConfig) ApiServerPort() (int, error) { +func (c *Config) ApiServerPort() (int, error) { var port string parsed, err := url.Parse(c.ApiServer.URL) @@ -390,7 +381,7 @@ func StringInList(s string, list []string) bool { return false } -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { +func (c *Config) ReadFromConfigFile(configFile string) error { contents, err := os.ReadFile(configFile) if err != nil { return fmt.Errorf("reading config file %q: %v", configFile, err) @@ -400,7 +391,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return fmt.Errorf("decoding config file %s: %v", configFile, err) } - // Wire new Config type to existing MicroshiftConfig + // Wire new Config type to existing Config c.Node = config.Node c.Debugging = config.Debugging c.Network = config.Network @@ -442,7 +433,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } -func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { +func (c *Config) computeAndUpdateClusterDNS() error { if len(c.Network.ServiceNetwork) == 0 { return fmt.Errorf("network.serviceNetwork not filled in") } @@ -457,7 +448,7 @@ func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { // Note: add a configFile parameter here because of unit test requiring custom // local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { +func (c *Config) ReadAndValidate(configFile string) error { if configFile != "" { if err := c.ReadFromConfigFile(configFile); err != nil { return err @@ -564,7 +555,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *MicroshiftConfig) GetVerbosity() int { +func (c *Config) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/assets/crd.go b/pkg/assets/crd.go index 392e2b9940..077f5f5507 100644 --- a/pkg/assets/crd.go +++ b/pkg/assets/crd.go @@ -70,7 +70,7 @@ func isEstablished(cs *apiextclientv1.ApiextensionsV1Client, obj apiruntime.Obje return false, err } -func WaitForCrdsEstablished(cfg *config.MicroshiftConfig) error { +func WaitForCrdsEstablished(cfg *config.Config) error { restConfig, err := clientcmd.BuildConfigFromFlags("", cfg.KubeConfigPath(config.KubeAdmin)) if err != nil { return err @@ -115,7 +115,7 @@ func applyCRD(client *apiextclientv1.ApiextensionsV1Client, crd *apiextv1.Custom return err } -func ApplyCRDs(cfg *config.MicroshiftConfig) error { +func ApplyCRDs(cfg *config.Config) error { lock.Lock() defer lock.Unlock() diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index d5faea3f98..745c835a34 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -36,7 +36,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func initCerts(cfg *config.Config) (*certchains.CertificateChains, error) { certChains, err := certSetup(cfg) if err != nil { return nil, err @@ -59,7 +59,7 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err return certChains, err } -func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func certSetup(cfg *config.Config) (*certchains.CertificateChains, error) { _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err @@ -367,7 +367,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func initKubeconfigs( - cfg *config.MicroshiftConfig, + cfg *config.Config, certChains *certchains.CertificateChains, ) error { inClusterTrustBundlePEM, err := os.ReadFile(cryptomaterial.ServiceAccountTokenCABundlePath(cryptomaterial.CertsDirectory(microshiftDataDir))) diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index cf9294a644..b8d72e3405 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -41,7 +41,7 @@ func NewRunMicroshiftCommand() *cobra.Command { return cmd } -func RunMicroshift(cfg *config.MicroshiftConfig) error { +func RunMicroshift(cfg *config.Config) error { if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { klog.Fatalf("Error in reading or validating configuration: %v", err) } diff --git a/pkg/components/components.go b/pkg/components/components.go index c20b5f2fc9..aaadbca36f 100755 --- a/pkg/components/components.go +++ b/pkg/components/components.go @@ -7,7 +7,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func StartComponents(cfg *config.MicroshiftConfig) error { +func StartComponents(cfg *config.Config) error { kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin) if err := startServiceCAController(cfg, kubeAdminConfig); err != nil { diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index b69f965cf7..13c4038980 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -9,7 +9,7 @@ import ( "k8s.io/klog/v2" ) -func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startServiceCAController(cfg *config.Config, kubeconfigPath string) error { var ( //TODO: fix the rolebinding and sa clusterRoleBinding = []string{ @@ -101,7 +101,7 @@ func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath strin return nil } -func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startIngressController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-router/cluster-role-binding.yaml", @@ -178,7 +178,7 @@ func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) return nil } -func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startDNSController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-dns/dns/cluster-role-binding.yaml", diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 0cefcd2e46..d0c75b2435 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -10,7 +10,7 @@ import ( "k8s.io/klog/v2" ) -func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCNIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/ovn/namespace.yaml", diff --git a/pkg/components/render.go b/pkg/components/render.go index b545d920de..5b466cc272 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -20,7 +20,7 @@ var templateFuncs = map[string]interface{}{ "Sha256sum": func(s string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) }, } -func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { +func renderParamsFromConfig(cfg *config.Config, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, "NodeName": cfg.Node.HostnameOverride, diff --git a/pkg/components/storage.go b/pkg/components/storage.go index 8a02e68350..816a0cbe11 100644 --- a/pkg/components/storage.go +++ b/pkg/components/storage.go @@ -24,7 +24,7 @@ func getCSIPluginConfig() (*lvmd.Lvmd, error) { return (&lvmd.Lvmd{}).WithDefaults(), nil } -func startCSIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCSIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/lvms/topolvm-openshift-storage_namespace.yaml", diff --git a/pkg/config/config.go b/pkg/config/config.go index 89bc81cb58..5f969305c1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -70,25 +70,16 @@ type EtcdConfig struct { DoStartupDefrag bool `json:"doStartupDefrag"` } -type MicroshiftConfig struct { - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` - - DNS DNS `json:"-"` - Node Node `json:"-"` - Debugging Debugging `json:"debugging"` - ApiServer ApiServer `json:"-"` - Network Network `json:"-"` -} - -// Top level config file type Config struct { DNS DNS `json:"dns"` Network Network `json:"network"` Node Node `json:"node"` ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` + + // Internal-only fields + Ingress IngressConfig `json:"-"` } type Network struct { @@ -189,11 +180,11 @@ const ( ) // KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { return filepath.Join(dataDir, "resources", string(id), "kubeconfig") } -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { +func (cfg *Config) KubeConfigAdminPath(id string) string { return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") } @@ -213,7 +204,7 @@ func getAllHostnames() ([]string, error) { return set.List(), nil } -func NewMicroshiftConfig() *MicroshiftConfig { +func NewMicroshiftConfig() *Config { nodeName, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -227,7 +218,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { klog.Fatalf("failed to get all hostnames: %v", err) } - return &MicroshiftConfig{ + return &Config{ Debugging: Debugging{ LogLevel: "Normal", }, @@ -268,7 +259,7 @@ func NewMicroshiftConfig() *MicroshiftConfig { } // Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { +func (c *Config) isDefaultNodeName() bool { hostname, err := os.Hostname() if err != nil { klog.Fatalf("Failed to get hostname %v", err) @@ -277,7 +268,7 @@ func (c *MicroshiftConfig) isDefaultNodeName() bool { } // Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { +func (c *Config) establishNodeName() (string, error) { filePath := filepath.Join(GetDataDir(), ".nodename") contents, err := os.ReadFile(filePath) if os.IsNotExist(err) { @@ -294,7 +285,7 @@ func (c *MicroshiftConfig) establishNodeName() (string, error) { } // Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { +func (c *Config) validateNodeName(isDefaultNodeName bool) error { if addr := net.ParseIP(c.Node.HostnameOverride); addr != nil { return fmt.Errorf("NodeName can not be an IP address: %q", c.Node.HostnameOverride) } @@ -319,7 +310,7 @@ func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { } // extract the api server port from the cluster URL -func (c *MicroshiftConfig) ApiServerPort() (int, error) { +func (c *Config) ApiServerPort() (int, error) { var port string parsed, err := url.Parse(c.ApiServer.URL) @@ -384,7 +375,7 @@ func StringInList(s string, list []string) bool { return false } -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { +func (c *Config) ReadFromConfigFile(configFile string) error { contents, err := os.ReadFile(configFile) if err != nil { return fmt.Errorf("reading config file %q: %v", configFile, err) @@ -394,7 +385,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return fmt.Errorf("decoding config file %s: %v", configFile, err) } - // Wire new Config type to existing MicroshiftConfig + // Wire new Config type to existing Config c.Node = config.Node c.Debugging = config.Debugging c.Network = config.Network @@ -436,7 +427,7 @@ func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { return nil } -func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { +func (c *Config) computeAndUpdateClusterDNS() error { if len(c.Network.ServiceNetwork) == 0 { return fmt.Errorf("network.serviceNetwork not filled in") } @@ -451,7 +442,7 @@ func (c *MicroshiftConfig) computeAndUpdateClusterDNS() error { // Note: add a configFile parameter here because of unit test requiring custom // local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { +func (c *Config) ReadAndValidate(configFile string) error { if configFile != "" { if err := c.ReadFromConfigFile(configFile); err != nil { return err @@ -558,7 +549,7 @@ func stringSliceContains(list []string, elements ...string) bool { } // GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *MicroshiftConfig) GetVerbosity() int { +func (c *Config) GetVerbosity() int { var verbosity int switch c.Debugging.LogLevel { case "Normal": diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 999ad3bc4b..a63df0a095 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,7 +30,7 @@ func setupSuiteDataDir(t *testing.T) func() { func TestConfigFile(t *testing.T) { var ttests = []struct { config Config - expected MicroshiftConfig + expected Config expectErr bool }{ { @@ -66,7 +66,7 @@ func TestConfigFile(t *testing.T) { DoStartupDefrag: true, }, }, - expected: MicroshiftConfig{ + expected: Config{ Debugging: Debugging{ LogLevel: "Debug", }, @@ -145,7 +145,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { var ttests = []struct { name string config Config - expected MicroshiftConfig + expected Config expectErr bool }{ { @@ -182,7 +182,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { DoStartupDefrag: true, }, }, - expected: MicroshiftConfig{ + expected: Config{ Debugging: Debugging{ LogLevel: "Debug", }, @@ -229,7 +229,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"127.0.0.1", "localhost"}, }, }, - expected: MicroshiftConfig{}, + expected: Config{}, expectErr: true, }, { @@ -239,7 +239,7 @@ func TestMicroshiftConfigReadAndValidate(t *testing.T) { SubjectAltNames: []string{"kubernetes"}, }, }, - expected: MicroshiftConfig{}, + expected: Config{}, expectErr: true, }, } diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index d94fc54de0..45ae5e9f7f 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -35,7 +35,7 @@ type ClusterPolicyController struct { configErr error } -func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyController { +func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { s := &ClusterPolicyController{} s.configErr = s.configure(cfg) return s @@ -44,7 +44,7 @@ func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyCont func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *ClusterPolicyController) configure(cfg *config.MicroshiftConfig) error { +func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) scheme := runtime.NewScheme() diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index 5596e8cb25..d68b99219f 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -38,7 +38,7 @@ var ( type EtcdService struct{} -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { return &EtcdService{} } diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 08a5c85298..7c5f8af766 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -26,10 +26,10 @@ import ( ) type InfrastructureServicesManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewInfrastructureServices(cfg *config.MicroshiftConfig) *InfrastructureServicesManager { +func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManager { s := &InfrastructureServicesManager{} s.cfg = cfg return s @@ -62,7 +62,7 @@ func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- st return ctx.Err() } -func applyDefaultRBACs(cfg *config.MicroshiftConfig) error { +func applyDefaultRBACs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( cr = []string{ diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 16e9db4f61..e46bab8d3f 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -80,7 +80,7 @@ type KubeAPIServer struct { advertiseAddress string } -func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { +func NewKubeAPIServer(cfg *config.Config) *KubeAPIServer { s := &KubeAPIServer{} if err := s.configure(cfg); err != nil { s.configureErr = err @@ -91,7 +91,7 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } -func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configure(cfg *config.Config) error { s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -255,7 +255,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { return nil } -func (s *KubeAPIServer) configureAuditPolicy(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configureAuditPolicy(cfg *config.Config) error { data := []byte(` apiVersion: audit.k8s.io/v1 kind: Policy diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index 70ebdff2ac..c256291fb0 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -49,7 +49,7 @@ type KubeControllerManager struct { configureErr error } -func NewKubeControllerManager(cfg *config.MicroshiftConfig) *KubeControllerManager { +func NewKubeControllerManager(cfg *config.Config) *KubeControllerManager { s := &KubeControllerManager{} // TODO: manage and invoke the configure bits independently outside of this. s.args, s.applyFn, s.configureErr = configure(cfg) @@ -74,7 +74,7 @@ func kcmServiceAccountPrivateKeyFile() string { return microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key" } -func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() error, err error) { +func configure(cfg *config.Config) (args []string, applyFn func() error, err error) { kubeConfig := cfg.KubeConfigPath(config.KubeControllerManager) clusterSigningKey, clusterSigningCert := kcmClusterSigningCertKeyAndFile() diff --git a/pkg/controllers/kube-scheduler.go b/pkg/controllers/kube-scheduler.go index 5b0889fa54..7c4ff950a8 100644 --- a/pkg/controllers/kube-scheduler.go +++ b/pkg/controllers/kube-scheduler.go @@ -40,7 +40,7 @@ type KubeScheduler struct { kubeconfig string } -func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { +func NewKubeScheduler(cfg *config.Config) *KubeScheduler { s := &KubeScheduler{} s.configure(cfg) return s @@ -49,7 +49,7 @@ func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { func (s *KubeScheduler) Name() string { return "kube-scheduler" } func (s *KubeScheduler) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { +func (s *KubeScheduler) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("failed to write kube-scheduler config: %v", err) } @@ -61,7 +61,7 @@ func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { s.kubeconfig = cfg.KubeConfigPath(config.KubeScheduler) } -func (s *KubeScheduler) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeScheduler) writeConfig(cfg *config.Config) error { data := []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: diff --git a/pkg/controllers/openshift-crd-manager.go b/pkg/controllers/openshift-crd-manager.go index f505b650d0..de328afc89 100644 --- a/pkg/controllers/openshift-crd-manager.go +++ b/pkg/controllers/openshift-crd-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftCRDManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftCRDManager(cfg *config.MicroshiftConfig) *OpenShiftCRDManager { +func NewOpenShiftCRDManager(cfg *config.Config) *OpenShiftCRDManager { s := &OpenShiftCRDManager{} s.cfg = cfg return s diff --git a/pkg/controllers/openshift-default-scc-manager.go b/pkg/controllers/openshift-default-scc-manager.go index d933f2015a..19615724c1 100644 --- a/pkg/controllers/openshift-default-scc-manager.go +++ b/pkg/controllers/openshift-default-scc-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftDefaultSCCManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftDefaultSCCManager(cfg *config.MicroshiftConfig) *OpenShiftDefaultSCCManager { +func NewOpenShiftDefaultSCCManager(cfg *config.Config) *OpenShiftDefaultSCCManager { s := &OpenShiftDefaultSCCManager{} s.cfg = cfg return s @@ -51,7 +51,7 @@ func (s *OpenShiftDefaultSCCManager) Run(ctx context.Context, ready chan<- struc return ctx.Err() } -func ApplyDefaultSCCs(cfg *config.MicroshiftConfig) error { +func ApplyDefaultSCCs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( clusterRole = []string{ diff --git a/pkg/controllers/openshift-route-controller-manager.go b/pkg/controllers/openshift-route-controller-manager.go index bf7df109f0..3bfa301fab 100644 --- a/pkg/controllers/openshift-route-controller-manager.go +++ b/pkg/controllers/openshift-route-controller-manager.go @@ -44,7 +44,7 @@ const ( componentRCM = "route-controller-manager" ) -func NewRouteControllerManager(cfg *config.MicroshiftConfig) *OCPRouteControllerManager { +func NewRouteControllerManager(cfg *config.Config) *OCPRouteControllerManager { s := &OCPRouteControllerManager{} s.configure(cfg) return s @@ -55,13 +55,13 @@ func (s *OCPRouteControllerManager) Dependencies() []string { return []string{"kube-apiserver", "openshift-crd-manager"} } -func (s *OCPRouteControllerManager) configure(cfg *config.MicroshiftConfig) { +func (s *OCPRouteControllerManager) configure(cfg *config.Config) { s.kubeconfig = cfg.KubeConfigPath(config.RouteControllerManager) s.kubeadmconfig = cfg.KubeConfigPath(config.KubeAdmin) s.config = s.writeConfig(cfg) } -func (s *OCPRouteControllerManager) writeConfig(cfg *config.MicroshiftConfig) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { +func (s *OCPRouteControllerManager) writeConfig(cfg *config.Config) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { servingCertDir := cryptomaterial.RouteControllerManagerServingCertDir(cryptomaterial.CertsDirectory(microshiftDataDir)) c := &openshiftcontrolplanev1.OpenShiftControllerManagerConfig{ diff --git a/pkg/controllers/version.go b/pkg/controllers/version.go index adb7e04c58..4495cbcc8f 100644 --- a/pkg/controllers/version.go +++ b/pkg/controllers/version.go @@ -25,10 +25,10 @@ import ( ) type VersionManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewVersionManager(cfg *config.MicroshiftConfig) *VersionManager { +func NewVersionManager(cfg *config.Config) *VersionManager { s := &VersionManager{} s.cfg = cfg return s diff --git a/pkg/kustomize/apply.go b/pkg/kustomize/apply.go index a1019f17e1..19a0b4bd3a 100644 --- a/pkg/kustomize/apply.go +++ b/pkg/kustomize/apply.go @@ -30,7 +30,7 @@ type Kustomizer struct { kubeconfig string } -func NewKustomizer(cfg *config.MicroshiftConfig) *Kustomizer { +func NewKustomizer(cfg *config.Config) *Kustomizer { return &Kustomizer{ paths: microshiftManifestsDir, kubeconfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 2a5d9a0ad3..3b94100ef6 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -34,7 +34,7 @@ type LoadbalancerServiceController struct { var _ servicemanager.Service = &LoadbalancerServiceController{} -func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { +func NewLoadbalancerServiceController(cfg *config.Config) *LoadbalancerServiceController { return &LoadbalancerServiceController{ NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 3daee405c2..29d7231aaf 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -23,7 +23,7 @@ type MicroShiftmDNSController struct { stopCh chan struct{} } -func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { +func NewMicroShiftmDNSController(cfg *config.Config) *MicroShiftmDNSController { return &MicroShiftmDNSController{ NodeIP: cfg.Node.NodeIP, NodeName: cfg.Node.HostnameOverride, diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index ebc246ded8..76bc0853b8 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -52,7 +52,7 @@ type KubeletServer struct { kubeconfig *kubeletconfig.KubeletConfiguration } -func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { +func NewKubeletServer(cfg *config.Config) *KubeletServer { s := &KubeletServer{} s.configure(cfg) return s @@ -61,7 +61,7 @@ func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { func (s *KubeletServer) Name() string { return componentKubelet } func (s *KubeletServer) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { +func (s *KubeletServer) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("Failed to write kubelet config", err) @@ -94,7 +94,7 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { s.kubeletflags = kubeletFlags } -func (s *KubeletServer) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeletServer) writeConfig(cfg *config.Config) error { certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) servingCertDir := cryptomaterial.KubeletServingCertDir(certsDir) diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e92f9c81d3..a202bc5a04 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -38,7 +38,7 @@ type NetworkConfiguration struct { skipInterfaceConfiguration bool } -func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration { +func NewNetworkConfiguration(cfg *config.Config) *NetworkConfiguration { n := &NetworkConfiguration{} n.configure(cfg) return n @@ -47,7 +47,7 @@ func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration func (n *NetworkConfiguration) Name() string { return componentNetworkConfiguration } func (n *NetworkConfiguration) Dependencies() []string { return []string{} } -func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { +func (n *NetworkConfiguration) configure(cfg *config.Config) { n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index 0c18891ff7..02f43e7018 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -36,7 +36,7 @@ type SysConfWatchController struct { timerFd int } -func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchController { +func NewSysConfWatchController(cfg *config.Config) *SysConfWatchController { // Create a realtime clock timer with asynchronous read support fd, err := unix.TimerfdCreate(unix.CLOCK_REALTIME, unix.TFD_CLOEXEC|unix.TFD_NONBLOCK) if err != nil {