diff --git a/etcd/cmd/microshift-etcd/run.go b/etcd/cmd/microshift-etcd/run.go index 0ea60ef6ec..a9a56de9c9 100644 --- a/etcd/cmd/microshift-etcd/run.go +++ b/etcd/cmd/microshift-etcd/run.go @@ -22,11 +22,11 @@ import ( ) func NewRunEtcdCommand() *cobra.Command { - cfg := config.NewMicroshiftConfig() cmd := &cobra.Command{ Use: "run", RunE: func(cmd *cobra.Command, args []string) (err error) { - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { + cfg, err := config.ActiveConfig() + if err != nil { klog.Fatalf("Error in reading and validating MicroShift config: %v", err) } @@ -52,10 +52,9 @@ type EtcdService struct { minDefragBytes int64 maxFragmentedPercentage float64 defragCheckFreq time.Duration - doStartupDefrag bool } -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { s := &EtcdService{} s.configure(cfg) return s @@ -63,11 +62,10 @@ func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { func (s *EtcdService) Name() string { return "etcd" } -func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { +func (s *EtcdService) configure(cfg *config.Config) { s.minDefragBytes = cfg.Etcd.MinDefragBytes s.maxFragmentedPercentage = cfg.Etcd.MaxFragmentedPercentage s.defragCheckFreq = cfg.Etcd.DefragCheckFreq - s.doStartupDefrag = cfg.Etcd.DoStartupDefrag microshiftDataDir := config.GetDataDir() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -92,8 +90,8 @@ func (s *EtcdService) configure(cfg *config.MicroshiftConfig) { s.etcdCfg.LCUrls = url2379 s.etcdCfg.ListenMetricsUrls = setURL([]string{"localhost"}, "2381") - s.etcdCfg.Name = cfg.NodeName - s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.NodeName, "localhost") + s.etcdCfg.Name = cfg.Node.HostnameOverride + s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.Node.HostnameOverride, "localhost") s.etcdCfg.CipherSuites = tlsCipherSuites s.etcdCfg.ClientTLSInfo.CertFile = cryptomaterial.PeerCertPath(etcdServingCertDir) @@ -120,13 +118,11 @@ func (s *EtcdService) Run() error { <-e.Server.StopNotify() }() - // If we were told to, go ahead and do a defragment now. - if s.doStartupDefrag { - if err := e.Server.Backend().Defrag(); err != nil { - err = fmt.Errorf("initial defragmentation failed: %v", err) - klog.Error(err) - return err - } + // Go ahead and do a defragment now. + if err := e.Server.Backend().Defrag(); err != nil { + err = fmt.Errorf("initial defragmentation failed: %v", err) + klog.Error(err) + return err } // Start up the defrag controller. diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go new file mode 100644 index 0000000000..88623d03e9 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/apiserver.go @@ -0,0 +1,19 @@ +package config + +type ApiServer struct { + // SubjectAltNames added to API server certs + SubjectAltNames []string `json:"subjectAltNames"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` + + // The URL and Port of the API server cannot be changed by the user. + URL string `json:"-"` + Port int `json:"-"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go index 97f9847e1c..26e0c71b40 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/config.go @@ -2,458 +2,191 @@ package config import ( "bytes" - "errors" "fmt" "net" "net/url" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/mitchellh/go-homedir" - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/component-base/logs" "k8s.io/klog/v2" - "sigs.k8s.io/yaml" + "github.com/apparentlymart/go-cidr/cidr" "github.com/openshift/microshift/pkg/util" ) const ( - DefaultUserConfigFile = "~/.microshift/config.yaml" - defaultUserDataDir = "~/.microshift/data" - DefaultGlobalConfigFile = "/etc/microshift/config.yaml" - defaultGlobalDataDir = "/var/lib/microshift" - // for files managed via management system in /etc, i.e. user applications - defaultManifestDirEtc = "/etc/microshift/manifests" - // for files embedded in ostree. i.e. cni/other component customizations - defaultManifestDirLib = "/usr/lib/microshift/manifests" // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) -var ( - configFile = findConfigFile() - dataDir = findDataDir() - manifestsDir = findManifestsDir() -) - -type ClusterConfig struct { - URL string `json:"-"` - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` -} - -type IngressConfig struct { - ServingCertificate []byte - ServingKey []byte -} - -type EtcdConfig struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimit uint64 - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - -type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - - SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` - Cluster ClusterConfig `json:"cluster"` - - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` -} - -// Top level config file type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` - Etcd Etcd `json:"etcd"` -} - -const ( - // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. - EtcdMinimumMemoryLimit = 50 -) - -type Etcd struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimitMB uint64 `json:"memoryLimitMB"` -} - -type Network struct { - // IP address pool to use for pod IPs. - // This field is immutable after installation. - ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork,omitempty"` - - // IP address pool for services. - // Currently, we only support a single entry here. - // This field is immutable after installation. - ServiceNetwork []string `json:"serviceNetwork,omitempty"` - - // The port range allowed for Services of type NodePort. - // If not specified, the default of 30000-32767 will be used. - // Such Services without a NodePort specified will have one - // automatically allocated from this range. - // This parameter can be updated after the cluster is - // installed. - // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` -} - -type ClusterNetworkEntry struct { - // The complete block for pod IPs. - CIDR string `json:"cidr,omitempty"` -} - -type DNS struct { - // baseDomain is the base domain of the cluster. All managed DNS records will - // be sub-domains of this base. - // - // For example, given the base domain `example.com`, router exposed - // domains will be formed as `*.apps.example.com` by default, - // and API service will have a DNS entry for `api.example.com`, - // as well as "api-int.example.com" for internal k8s API access. - // - // Once set, this field cannot be changed. - BaseDomain string `json:"baseDomain"` -} - -type ApiServer struct { - // SubjectAltNames added to API server certs - SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` -} - -type Node struct { - // If non-empty, will use this string to identify the node instead of the hostname - HostnameOverride string `json:"hostnameOverride"` - - // IP address of the node, passed to the kubelet. - // If not specified, kubelet will use the node's default IP address. - NodeIP string `json:"nodeIP"` -} - -type Debugging struct { - // Valid values are: "Normal", "Debug", "Trace", "TraceAll". - // Defaults to "Normal". - LogLevel string `json:"logLevel"` -} - -func GetConfigFile() string { - return configFile -} - -func GetDataDir() string { - return dataDir -} + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` -func GetManifestsDir() []string { - return manifestsDir + // Internal-only fields + Ingress IngressConfig `json:"-"` + userSettings *Config `json:"-"` // the values read from the config file } -// KubeConfigID identifies the different kubeconfigs managed in the DataDir -type KubeConfigID string - -const ( - KubeAdmin KubeConfigID = "kubeadmin" - KubeControllerManager KubeConfigID = "kube-controller-manager" - KubeScheduler KubeConfigID = "kube-scheduler" - Kubelet KubeConfigID = "kubelet" - ClusterPolicyController KubeConfigID = "cluster-policy-controller" - RouteControllerManager KubeConfigID = "route-controller-manager" -) - -// KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { - return filepath.Join(dataDir, "resources", string(id), "kubeconfig") -} - -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { - return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") -} - -var allHostnames []string - -func getAllHostnames() ([]string, error) { - if len(allHostnames) != 0 { - return allHostnames, nil +// NewDefault creates a new Config struct populated with the +// default values and with any computed values updated based on those +// defaults. +func NewDefault() *Config { + c := &Config{} + if err := c.fillDefaults(); err != nil { + klog.Fatalf("Failed to initialize config: %v", err) } - cmd := exec.Command("/bin/hostname", "-A") - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - return nil, fmt.Errorf("Error when executing 'hostname -A': %v", err) + if err := c.updateComputedValues(); err != nil { + klog.Fatalf("Failed to initialize config: %v", err) } - outString := out.String() - outString = strings.Trim(outString[:len(outString)-1], " ") - // Remove duplicates to avoid having them in the certificates. - names := strings.Split(outString, " ") - set := sets.NewString(names...) - allHostnames = set.List() - return allHostnames, nil + return c } -func NewMicroshiftConfig() *MicroshiftConfig { - nodeName, err := os.Hostname() +// fillDefaults forcibly sets the configuration to the default +// values. We do not use a static struct for the defaults because some +// of them are computed from the environment. If any error occurs +// probing the environment, the values in the Config instance are not +// changed. +func (c *Config) fillDefaults() error { + + // Look up any values that may generate an error + subjectAltNames, err := getAllHostnames() if err != nil { - klog.Fatalf("Failed to get hostname %v", err) + return fmt.Errorf("failed to get all hostnames: %v", err) } - nodeIP, err := util.GetHostIP() + hostname, err := os.Hostname() if err != nil { - klog.Fatalf("failed to get host IP: %v", err) + return fmt.Errorf("Failed to get hostname %v", err) } - subjectAltNames, err := getAllHostnames() + nodeIP, err := util.GetHostIP() if err != nil { - klog.Fatalf("failed to get all hostnames: %v", err) + return fmt.Errorf("failed to get host IP: %v", err) } - return &MicroshiftConfig{ - LogVLevel: 2, - SubjectAltNames: subjectAltNames, - NodeName: strings.ToLower(nodeName), - NodeIP: nodeIP, - BaseDomain: "example.com", - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", - ServiceNodePortRange: "30000-32767", - }, - Etcd: EtcdConfig{ - MemoryLimit: 0, // No limit - MinDefragBytes: 100 * 1024 * 1024, // 100MB - MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, // 8GB - }, + c.Debugging = Debugging{ + LogLevel: "Normal", } -} - -// Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { - hostname, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) + c.ApiServer = ApiServer{ + SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", + Port: 6443, } - return c.NodeName == strings.ToLower(hostname) -} - -// Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { - filePath := filepath.Join(GetDataDir(), ".nodename") - contents, err := os.ReadFile(filePath) - if os.IsNotExist(err) { - // ensure that dataDir exists - os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { - return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) - } - return c.NodeName, nil - } else if err != nil { - return "", err + c.Node = Node{ + HostnameOverride: hostname, + NodeIP: nodeIP, } - return string(contents), nil -} - -// Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + c.DNS = DNS{ + BaseDomain: "example.com", } - - establishedNodeName, err := c.establishNodeName() - if err != nil { - return fmt.Errorf("failed to establish NodeName: %v", err) + c.Network = Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, + ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", } - - if establishedNodeName != c.NodeName { - if !isDefaultNodeName { - return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) - } else { - c.NodeName = establishedNodeName - klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) - } + c.Etcd = EtcdConfig{ + MemoryLimitMB: 0, + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, + MinDefragBytes: 100 * 1024 * 1024, + MaxFragmentedPercentage: 45, + DefragCheckFreq: 5 * time.Minute, } return nil } -// extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { - var port string +// incorporateUserSettings merges any values read from the +// configuration file provided by the user with the existing settings +// (usually the defaults). +func (c *Config) incorporateUserSettings(u *Config) { + c.userSettings = u - parsed, err := url.Parse(c.URL) - if err != nil { - return 0, err + if u.DNS.BaseDomain != "" { + c.DNS.BaseDomain = u.DNS.BaseDomain } - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err + if len(u.Network.ClusterNetwork) != 0 { + c.Network.ClusterNetwork = u.Network.ClusterNetwork } - return portNum, nil -} - -// Returns the default user config file if that exists, else the default global -// config file, else the empty string. -func findConfigFile() string { - userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) - if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { - if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { - return "" - } else { - return DefaultGlobalConfigFile + if len(u.Network.ServiceNetwork) != 0 { + c.Network.ServiceNetwork = u.Network.ServiceNetwork + // The default for the API server address is computed from the + // service network. If the user provides a network without + // also overriding the computed address, we need to clear the + // address here so it is recomputed later. If they provide + // both the network and the address, the address will be + // copied into place below with the other API server settings. + if u.ApiServer.AdvertiseAddress == "" { + c.ApiServer.AdvertiseAddress = "" } - } else { - return userConfigFile } -} - -// Returns the default user data dir if it exists or the user is non-root. -// Returns the default global data dir otherwise. -func findDataDir() string { - userDataDir, _ := homedir.Expand(defaultUserDataDir) - if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { - if os.Geteuid() > 0 { - return userDataDir - } else { - return defaultGlobalDataDir - } - } else { - return userDataDir + if u.Network.ServiceNodePortRange != "" { + c.Network.ServiceNodePortRange = u.Network.ServiceNodePortRange } -} - -// Returns the default manifests directories -func findManifestsDir() []string { - var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} - return manifestsDir -} - -func StringInList(s string, list []string) bool { - for _, x := range list { - if x == s { - return true - } + if u.Network.DNS != "" { + c.Network.DNS = u.Network.DNS } - return false -} -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) - } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) + if u.Etcd.MemoryLimitMB != 0 { + c.Etcd.MemoryLimitMB = u.Etcd.MemoryLimitMB } - // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = strings.ToLower(config.Node.HostnameOverride) + if u.Node.HostnameOverride != "" { + c.Node.HostnameOverride = u.Node.HostnameOverride } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP + if u.Node.NodeIP != "" { + c.Node.NodeIP = u.Node.NodeIP } - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange - } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain + + if len(u.ApiServer.SubjectAltNames) != 0 { + c.ApiServer.SubjectAltNames = u.ApiServer.SubjectAltNames } - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames + if u.ApiServer.AdvertiseAddress != "" { + c.ApiServer.AdvertiseAddress = u.ApiServer.AdvertiseAddress } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress + if u.ApiServer.URL != "" { + c.ApiServer.URL = u.ApiServer.URL } - if config.Etcd.MemoryLimitMB > 0 { - // If the memory limit is than the minimum, set it to the minimum and continue. - if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit - } else { - c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB - } + if u.Debugging.LogLevel != "" { + c.Debugging.LogLevel = u.Debugging.LogLevel } - - return nil } -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } +// updateComputedValues examins the existing settings and converts any +// inputs to more easily consumable units or fills in any defaults +// computed based on the values of other settings. +func (c *Config) updateComputedValues() error { - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) + clusterDNS, err := c.computeClusterDNS() if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + return err } - c.Cluster.DNS = clusterDNS + c.Network.DNS = clusterDNS - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + // If KAS advertise address configured, we do not want to apply + // the IP to the internal interface. + if c.userSettings != nil && len(c.userSettings.ApiServer.AdvertiseAddress) != 0 { + c.ApiServer.SkipInterface = true + } + + // If we have no advertise address, pick one. + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) // Since the KAS advertise address was not provided we will default to the // next immediate subnet after the service CIDR. This is due to the fact // that using the actual apiserver service IP as an endpoint slice breaks @@ -466,13 +199,14 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } // First and last are the same because of the /32 netmask. firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.KASAdvertiseAddress = firstValidIP.String() - c.SkipKASInterface = false - } else { - c.SkipKASInterface = true + c.ApiServer.AdvertiseAddress = firstValidIP.String() } - if len(c.SubjectAltNames) > 0 { + return nil +} + +func (c *Config) validate() error { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -485,25 +219,24 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } - if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -512,76 +245,39 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) + + if c.Etcd.MemoryLimitMB > 0 && c.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + return fmt.Errorf("etcd.memoryLimitMB value %d is below the minimum allowed %d", + c.Etcd.MemoryLimitMB, EtcdMinimumMemoryLimit, + ) } return nil } -// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork -func getClusterDNS(serviceCIDR string) (string, error) { - _, service, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) - } - dnsClusterIP, err := cidr.Host(service, 10) - if err != nil { - return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) - } - - return dnsClusterIP.String(), nil -} +var allHostnames []string -func stringSliceContains(list []string, elements ...string) bool { - for _, value := range list { - for _, element := range elements { - if value == element { - return true - } - } +func getAllHostnames() ([]string, error) { + if len(allHostnames) != 0 { + return allHostnames, nil } - return false -} - -// GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { - var verbosity int - switch c.Debugging.LogLevel { - case "Normal": - verbosity = 2 - case "Debug": - verbosity = 4 - case "Trace": - verbosity = 6 - case "TraceAll": - verbosity = 8 - default: - verbosity = 2 + cmd := exec.Command("/bin/hostname", "-A") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("Error when executing 'hostname -A': %v", err) } - return verbosity -} - -func HideUnsupportedFlags(flags *pflag.FlagSet) { - // hide logging flags that we do not use/support - loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) - logs.AddFlags(loggingFlags) - - supportedLoggingFlags := sets.NewString("v") - - loggingFlags.VisitAll(func(pf *pflag.Flag) { - if !supportedLoggingFlags.Has(pf.Name) { - flags.MarkHidden(pf.Name) - } - }) - - flags.MarkHidden("version") + outString := out.String() + outString = strings.Trim(outString[:len(outString)-1], " ") + // Remove duplicates to avoid having them in the certificates. + names := strings.Split(outString, " ") + set := sets.NewString(names...) + allHostnames = set.List() + return allHostnames, nil } diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go new file mode 100644 index 0000000000..a540f5490c --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/debugging.go @@ -0,0 +1,25 @@ +package config + +type Debugging struct { + // Valid values are: "Normal", "Debug", "Trace", "TraceAll". + // Defaults to "Normal". + LogLevel string `json:"logLevel"` +} + +// GetVerbosity returns the numerical value for LogLevel which is an enum +func (c *Config) GetVerbosity() int { + var verbosity int + switch c.Debugging.LogLevel { + case "Normal": + verbosity = 2 + case "Debug": + verbosity = 4 + case "Trace": + verbosity = 6 + case "TraceAll": + verbosity = 8 + default: + verbosity = 2 + } + return verbosity +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go new file mode 100644 index 0000000000..ca6f11e88e --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/dns.go @@ -0,0 +1,14 @@ +package config + +type DNS struct { + // baseDomain is the base domain of the cluster. All managed DNS records will + // be sub-domains of this base. + // + // For example, given the base domain `example.com`, router exposed + // domains will be formed as `*.apps.example.com` by default, + // and API service will have a DNS entry for `api.example.com`, + // as well as "api-int.example.com" for internal k8s API access. + // + // Once set, this field cannot be changed. + BaseDomain string `json:"baseDomain"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go new file mode 100644 index 0000000000..cb571eb963 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/etcd.go @@ -0,0 +1,29 @@ +package config + +import "time" + +const ( + // Etcd performance degrades significantly if the memory available + // is less than 50MB, enforce this minimum. + EtcdMinimumMemoryLimit = 50 +) + +type EtcdConfig struct { + // Set a memory limit on the etcd process; etcd will begin paging + // memory when it gets to this value. 0 means no limit. + MemoryLimitMB uint64 `json:"memoryLimitMB"` + + // The limit on the size of the etcd database; etcd will start + // failing writes if its size on disk reaches this value + QuotaBackendBytes int64 `json:"-"` + + // If the backend is fragmented more than + // `maxFragmentedPercentage` and the database size is greater than + // `minDefragBytes`, do a defrag. + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"-"` + + // How often to check the conditions for defragging (0 means no + // defrags, except for a single on startup). + DefragCheckFreq time.Duration `json:"-"` +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go new file mode 100644 index 0000000000..6c78397125 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/files.go @@ -0,0 +1,126 @@ +package config + +import ( + "errors" + "fmt" + "os" + + "github.com/mitchellh/go-homedir" + "sigs.k8s.io/yaml" +) + +const ( + DefaultUserConfigFile = "~/.microshift/config.yaml" + defaultUserDataDir = "~/.microshift/data" + DefaultGlobalConfigFile = "/etc/microshift/config.yaml" + defaultGlobalDataDir = "/var/lib/microshift" + // for files managed via management system in /etc, i.e. user applications + defaultManifestDirEtc = "/etc/microshift/manifests" + // for files embedded in ostree. i.e. cni/other component customizations + defaultManifestDirLib = "/usr/lib/microshift/manifests" +) + +var ( + configFile = findConfigFile() + dataDir = findDataDir() + manifestsDir = findManifestsDir() +) + +func GetConfigFile() string { + return configFile +} + +func GetDataDir() string { + return dataDir +} + +func GetManifestsDir() []string { + return manifestsDir +} + +// Returns the default user config file if that exists, else the default global +// config file, else the empty string. +func findConfigFile() string { + userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) + if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { + return "" + } else { + return DefaultGlobalConfigFile + } + } else { + return userConfigFile + } +} + +// Returns the default user data dir if it exists or the user is non-root. +// Returns the default global data dir otherwise. +func findDataDir() string { + userDataDir, _ := homedir.Expand(defaultUserDataDir) + if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { + if os.Geteuid() > 0 { + return userDataDir + } else { + return defaultGlobalDataDir + } + } else { + return userDataDir + } +} + +// Returns the default manifests directories +func findManifestsDir() []string { + var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} + return manifestsDir +} + +func parse(contents []byte) (*Config, error) { + c := &Config{} + if err := yaml.Unmarshal(contents, c); err != nil { + return nil, fmt.Errorf("Unable to decode configuration: %v", err) + } + return c, nil +} + +func getActiveConfigFromYAML(contents []byte) (*Config, error) { + userSettings, err := parse(contents) + if err != nil { + return nil, fmt.Errorf("Error parsing config file %q: %v", configFile, err) + } + + // Start with the defaults, then apply the user settings and + // recompute dynamic values. + results := &Config{} + if err := results.fillDefaults(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + results.incorporateUserSettings(userSettings) + if err := results.updateComputedValues(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + if err := results.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + return results, nil +} + +// ActiveConfig returns the active configuration. If the configuration +// file exists, read it and require it to be valid. Otherwise return +// the default settings. +func ActiveConfig() (*Config, error) { + filename := GetConfigFile() + _, err := os.Stat(filename) + if os.IsNotExist(err) { + // No configuration file, use the default settings + return NewDefault(), nil + } else if err != nil { + return nil, err + } + + // Read the file and merge user-provided settings with the defaults + contents, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("Error reading config file %q: %v", configFile, err) + } + return getActiveConfigFromYAML(contents) +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go new file mode 100644 index 0000000000..e15ee75a1c --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/logs" +) + +func HideUnsupportedFlags(flags *pflag.FlagSet) { + // hide logging flags that we do not use/support + loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) + logs.AddFlags(loggingFlags) + + supportedLoggingFlags := sets.NewString("v") + + loggingFlags.VisitAll(func(pf *pflag.Flag) { + if !supportedLoggingFlags.Has(pf.Name) { + flags.MarkHidden(pf.Name) + } + }) + + flags.MarkHidden("version") +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go new file mode 100644 index 0000000000..a8980ae1ed --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/ingress.go @@ -0,0 +1,6 @@ +package config + +type IngressConfig struct { + ServingCertificate []byte + ServingKey []byte +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go new file mode 100644 index 0000000000..1e33b07b96 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/kubeconfig.go @@ -0,0 +1,24 @@ +package config + +import "path/filepath" + +// KubeConfigID identifies the different kubeconfigs managed in the DataDir +type KubeConfigID string + +const ( + KubeAdmin KubeConfigID = "kubeadmin" + KubeControllerManager KubeConfigID = "kube-controller-manager" + KubeScheduler KubeConfigID = "kube-scheduler" + Kubelet KubeConfigID = "kubelet" + ClusterPolicyController KubeConfigID = "cluster-policy-controller" + RouteControllerManager KubeConfigID = "route-controller-manager" +) + +// KubeConfigPath returns the path to the specified kubeconfig file. +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { + return filepath.Join(dataDir, "resources", string(id), "kubeconfig") +} + +func (cfg *Config) KubeConfigAdminPath(id string) string { + return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go new file mode 100644 index 0000000000..3a3f8cc211 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/network.go @@ -0,0 +1,62 @@ +package config + +import ( + "fmt" + "net" + + "github.com/apparentlymart/go-cidr/cidr" +) + +type Network struct { + // IP address pool to use for pod IPs. + // This field is immutable after installation. + ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork"` + + // IP address pool for services. + // Currently, we only support a single entry here. + // This field is immutable after installation. + ServiceNetwork []string `json:"serviceNetwork"` + + // The port range allowed for Services of type NodePort. + // If not specified, the default of 30000-32767 will be used. + // Such Services without a NodePort specified will have one + // automatically allocated from this range. + // This parameter can be updated after the cluster is + // installed. + // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` + ServiceNodePortRange string `json:"serviceNodePortRange"` + + // The DNS server to use + DNS string `json:"-"` +} + +type ClusterNetworkEntry struct { + // The complete block for pod IPs. + CIDR string `json:"cidr"` +} + +func (c *Config) computeClusterDNS() (string, error) { + if len(c.Network.ServiceNetwork) == 0 { + return "", fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return "", fmt.Errorf("failed to get DNS IP: %v", err) + } + return clusterDNS, nil +} + +// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork +func getClusterDNS(serviceCIDR string) (string, error) { + _, service, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) + } + dnsClusterIP, err := cidr.Host(service, 10) + if err != nil { + return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) + } + + return dnsClusterIP.String(), nil +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go new file mode 100644 index 0000000000..13eb6e6a90 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/node.go @@ -0,0 +1,87 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +type Node struct { + // If non-empty, will use this string to identify the node instead of the hostname + HostnameOverride string `json:"hostnameOverride"` + + // IP address of the node, passed to the kubelet. + // If not specified, kubelet will use the node's default IP address. + NodeIP string `json:"nodeIP"` +} + +// Determine if the config file specified a NodeName (by default it's assigned the hostname) +func (c *Config) isDefaultNodeName() bool { + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Failed to get hostname %v", err) + } + return c.CanonicalNodeName() == strings.ToLower(hostname) +} + +// CanonicalNodeName returns the name to use for the node. The value +// is taken from either the HostnameOverride provided by the user in +// the config file, or the host name. +func (c *Config) CanonicalNodeName() string { + return strings.ToLower(c.Node.HostnameOverride) +} + +// Read or set the NodeName that will be used for this MicroShift instance +func (c *Config) establishNodeName() (string, error) { + name := c.CanonicalNodeName() + filePath := filepath.Join(GetDataDir(), ".nodename") + contents, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + // ensure that dataDir exists + os.MkdirAll(GetDataDir(), 0700) + if err := os.WriteFile(filePath, []byte(name), 0444); err != nil { + return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) + } + return name, nil + } else if err != nil { + return "", err + } + return string(contents), nil +} + +// Validate the NodeName to be used for this MicroShift instances +func (c *Config) validateNodeName(isDefaultNodeName bool) error { + currentNodeName := c.CanonicalNodeName() + if addr := net.ParseIP(currentNodeName); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", currentNodeName) + } + + establishedNodeName, err := c.establishNodeName() + if err != nil { + return fmt.Errorf("failed to establish NodeName: %v", err) + } + + if establishedNodeName != currentNodeName { + if !isDefaultNodeName { + return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", + currentNodeName, establishedNodeName) + } else { + c.Node.HostnameOverride = establishedNodeName + klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ + "Please consider using a static NodeName in configuration", establishedNodeName) + } + } + + return nil +} + +func (c *Config) EnsureNodeNameHasNotChanged() error { + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + return c.validateNodeName(c.isDefaultNodeName()) +} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go new file mode 100644 index 0000000000..6bb69a5079 --- /dev/null +++ b/etcd/vendor/github.com/openshift/microshift/pkg/config/util.go @@ -0,0 +1,21 @@ +package config + +func StringInList(s string, list []string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func stringSliceContains(list []string, elements ...string) bool { + for _, value := range list { + for _, element := range elements { + if value == element { + return true + } + } + } + return false +} diff --git a/pkg/assets/crd.go b/pkg/assets/crd.go index 392e2b9940..077f5f5507 100644 --- a/pkg/assets/crd.go +++ b/pkg/assets/crd.go @@ -70,7 +70,7 @@ func isEstablished(cs *apiextclientv1.ApiextensionsV1Client, obj apiruntime.Obje return false, err } -func WaitForCrdsEstablished(cfg *config.MicroshiftConfig) error { +func WaitForCrdsEstablished(cfg *config.Config) error { restConfig, err := clientcmd.BuildConfigFromFlags("", cfg.KubeConfigPath(config.KubeAdmin)) if err != nil { return err @@ -115,7 +115,7 @@ func applyCRD(client *apiextclientv1.ApiextensionsV1Client, crd *apiextv1.Custom return err } -func ApplyCRDs(cfg *config.MicroshiftConfig) error { +func ApplyCRDs(cfg *config.Config) error { lock.Lock() defer lock.Unlock() diff --git a/pkg/cmd/init.go b/pkg/cmd/init.go index b3ab6c026e..864e785b0f 100644 --- a/pkg/cmd/init.go +++ b/pkg/cmd/init.go @@ -22,6 +22,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "time" "k8s.io/apiserver/pkg/authentication/serviceaccount" @@ -36,7 +37,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { +func initCerts(cfg *config.Config) (*certchains.CertificateChains, error) { certChains, err := certSetup(cfg) if err != nil { return nil, err @@ -59,8 +60,8 @@ func initCerts(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err return certChains, err } -func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, error) { - _, svcNet, err := net.ParseCIDR(cfg.Cluster.ServiceCIDR) +func certSetup(cfg *config.Config) (*certchains.CertificateChains, error) { + _, svcNet, err := net.ParseCIDR(cfg.Network.ServiceNetwork[0]) if err != nil { return nil, err } @@ -71,18 +72,18 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } externalCertNames := []string{ - cfg.NodeName, - "api." + cfg.BaseDomain, + cfg.Node.HostnameOverride, + "api." + cfg.DNS.BaseDomain, } - externalCertNames = append(externalCertNames, cfg.SubjectAltNames...) + externalCertNames = append(externalCertNames, cfg.ApiServer.SubjectAltNames...) // When Kube apiserver advertise address matches the node IP we can not add // it to the certificates or else the internal pod access to apiserver is // broken. Because of client-go not using SNI and the way apiserver handles // which certificate to serve which destination IP, internal pods start // getting the external certificate, which is signed by a different CA and // does not match the hostname. - if cfg.KASAdvertiseAddress != cfg.NodeIP { - externalCertNames = append(externalCertNames, cfg.NodeIP) + if cfg.ApiServer.AdvertiseAddress != cfg.Node.NodeIP { + externalCertNames = append(externalCertNames, cfg.Node.NodeIP) } certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) @@ -172,7 +173,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, // userinfo per https://kubernetes.io/docs/reference/access-authn-authz/node/#overview - UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.NodeName, Groups: []string{"system:nodes"}}, + UserInfo: &user.DefaultInfo{Name: "system:node:" + cfg.CanonicalNodeName(), Groups: []string{"system:nodes"}}, }, ).WithServingCertificates( &certchains.ServingCertificateSigningRequestInfo{ @@ -180,7 +181,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err Name: "kubelet-server", ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, - Hostnames: []string{cfg.NodeName, cfg.NodeIP}, + Hostnames: []string{cfg.Node.HostnameOverride, cfg.Node.NodeIP}, }, ), ), @@ -229,7 +230,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.ShortLivedCertificateValidityDays, }, Hostnames: []string{ - "*.apps." + cfg.BaseDomain, // wildcard for any additional auto-generated domains + "*.apps." + cfg.DNS.BaseDomain, // wildcard for any additional auto-generated domains }, }, ), @@ -285,8 +286,8 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - "api." + cfg.BaseDomain, - "api-int." + cfg.BaseDomain, + "api." + cfg.DNS.BaseDomain, + "api-int." + cfg.DNS.BaseDomain, apiServerServiceIP.String(), }, }, @@ -314,7 +315,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-peer:etcd-client", Groups: []string{"system:etcd-peers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, &certchains.PeerCertificateSigningRequestInfo{ CSRMeta: certchains.CSRMeta{ @@ -322,7 +323,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err ValidityDays: cryptomaterial.LongLivedCertificateValidityDays, }, UserInfo: &user.DefaultInfo{Name: "system:etcd-server:etcd-client", Groups: []string{"system:etcd-servers"}}, - Hostnames: []string{"localhost", cfg.NodeName}, + Hostnames: []string{"localhost", cfg.Node.HostnameOverride}, }, ), ).WithCABundle( @@ -366,7 +367,7 @@ func certSetup(cfg *config.MicroshiftConfig) (*certchains.CertificateChains, err } func initKubeconfigs( - cfg *config.MicroshiftConfig, + cfg *config.Config, certChains *certchains.CertificateChains, ) error { externalTrustPEM, err := os.ReadFile(cryptomaterial.CACertPath(cryptomaterial.KubeAPIServerExternalSigner(cryptomaterial.CertsDirectory(microshiftDataDir)))) @@ -383,18 +384,14 @@ func initKubeconfigs( return err } - u, err := url.Parse(cfg.Cluster.URL) + u, err := url.Parse(cfg.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } - apiServerPort, err := cfg.Cluster.ApiServerPort() - if err != nil { - return fmt.Errorf("failed to get apiserver port: %v", err) - } // Generate one kubeconfigs per name - for _, name := range append(cfg.SubjectAltNames, cfg.NodeName) { - u.Host = fmt.Sprintf("%s:%d", name, apiServerPort) + for _, name := range append(cfg.ApiServer.SubjectAltNames, cfg.Node.HostnameOverride) { + u.Host = net.JoinHostPort(name, strconv.Itoa(cfg.ApiServer.Port)) if err := util.KubeConfigWithClientCerts( cfg.KubeConfigAdminPath(name), u.String(), @@ -408,7 +405,7 @@ func initKubeconfigs( if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeAdmin), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, adminKubeconfigCertPEM, adminKubeconfigKeyPEM, @@ -422,7 +419,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, kcmCertPEM, kcmKeyPEM, @@ -436,7 +433,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.KubeScheduler), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, schedulerCertPEM, schedulerKeyPEM, ); err != nil { @@ -449,7 +446,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.Kubelet), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, kubeletCertPEM, kubeletKeyPEM, ); err != nil { @@ -461,7 +458,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.ClusterPolicyController), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, clusterPolicyControllerCertPEM, clusterPolicyControllerKeyPEM, ); err != nil { @@ -474,7 +471,7 @@ func initKubeconfigs( } if err := util.KubeConfigWithClientCerts( cfg.KubeConfigPath(config.RouteControllerManager), - cfg.Cluster.URL, + cfg.ApiServer.URL, internalTrustPEM, routeControllerManagerCertPEM, routeControllerManagerKeyPEM, ); err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index e8c3558862..b5fe89b9fa 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "time" @@ -21,6 +22,7 @@ import ( "github.com/spf13/cobra" "k8s.io/klog/v2" + "sigs.k8s.io/yaml" ) const ( @@ -28,12 +30,20 @@ const ( ) func NewRunMicroshiftCommand() *cobra.Command { - cfg := config.NewMicroshiftConfig() - cmd := &cobra.Command{ Use: "run", Short: "Run MicroShift", RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.ActiveConfig() + if err != nil { + return err + } + // Things to very badly if the node's name has changed + // since the last time the server started. + err = cfg.EnsureNodeNameHasNotChanged() + if err != nil { + return err + } return RunMicroshift(cfg) }, } @@ -41,28 +51,37 @@ func NewRunMicroshiftCommand() *cobra.Command { return cmd } -func RunMicroshift(cfg *config.MicroshiftConfig) error { - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { - klog.Fatalf("Error in reading or validating configuration: %v", err) +func logConfig(cfg *config.Config) { + marshalled, err := yaml.Marshal(cfg) + if err != nil { + klog.Fatal(err) } + klog.Info("Effective configuration:") + for _, line := range strings.Split(string(marshalled), "\n") { + klog.Info(line) + } +} +func RunMicroshift(cfg *config.Config) error { // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") } + logConfig(cfg) + // TO-DO: When multi-node is ready, we need to add the controller host-name/mDNS hostname // or VIP to this list on start // see https://github.com/openshift/microshift/pull/471 if err := util.AddToNoProxyEnv( - cfg.NodeIP, - cfg.NodeName, - cfg.Cluster.ClusterCIDR, - cfg.Cluster.ServiceCIDR, + cfg.Node.NodeIP, + cfg.Node.HostnameOverride, + cfg.Network.ClusterNetwork[0].CIDR, + cfg.Network.ServiceNetwork[0], ".svc", ".cluster.local", - "."+cfg.BaseDomain); err != nil { + "."+cfg.DNS.BaseDomain); err != nil { klog.Fatal(err) } diff --git a/pkg/cmd/showConfig.go b/pkg/cmd/showConfig.go index 21c57f5f08..1f92352fd5 100644 --- a/pkg/cmd/showConfig.go +++ b/pkg/cmd/showConfig.go @@ -3,13 +3,11 @@ package cmd import ( "fmt" + "github.com/openshift/microshift/pkg/config" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/yaml" - - "github.com/openshift/microshift/pkg/config" ) type showConfigOptions struct { @@ -22,54 +20,23 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command Mode: "default", } - cfg := config.NewMicroshiftConfig() - cmd := &cobra.Command{ Use: "show-config", Short: "Print MicroShift's configuration", Run: func(cmd *cobra.Command, args []string) { + var cfg *config.Config + var err error - switch opts.Mode { - case "default": - cfg.NodeIP = "" - cfg.NodeName = "" - case "effective": - // Load the current configuration - if err := cfg.ReadAndValidate(config.GetConfigFile()); err != nil { + if opts.Mode == "effective" { + cfg, err = config.ActiveConfig() + if err != nil { cmdutil.CheckErr(err) } - default: - cmdutil.CheckErr(fmt.Errorf("Unknown mode %q", opts.Mode)) + } else { + cfg = config.NewDefault() } - // map back from internal representation to user config - logLevels := []string{"", "", "Normal", "", "Debug", "", "Trace", "", "TraceAll"} - if cfg.LogVLevel < 0 || cfg.LogVLevel >= len(logLevels) { - klog.Fatal("logVLevel out of range [0..%d] %d", len(logLevels)-1, cfg.LogVLevel) - } - userCfg := config.Config{ - Network: config.Network{ - ClusterNetwork: []config.ClusterNetworkEntry{ - {CIDR: cfg.Cluster.ClusterCIDR}, - }, - ServiceNetwork: []string{cfg.Cluster.ServiceCIDR}, - ServiceNodePortRange: cfg.Cluster.ServiceNodePortRange, - }, - DNS: config.DNS{ - BaseDomain: cfg.BaseDomain, - }, - Node: config.Node{ - HostnameOverride: cfg.NodeName, - NodeIP: cfg.NodeIP, - }, - ApiServer: config.ApiServer{ - SubjectAltNames: cfg.SubjectAltNames, - }, - Debugging: config.Debugging{ - LogLevel: logLevels[cfg.LogVLevel], - }, - } - marshalled, err := yaml.Marshal(userCfg) + marshalled, err := yaml.Marshal(cfg) cmdutil.CheckErr(err) fmt.Fprintf(ioStreams.Out, "%s\n", string(marshalled)) diff --git a/pkg/components/components.go b/pkg/components/components.go index c20b5f2fc9..aaadbca36f 100755 --- a/pkg/components/components.go +++ b/pkg/components/components.go @@ -7,7 +7,7 @@ import ( var microshiftDataDir = config.GetDataDir() -func StartComponents(cfg *config.MicroshiftConfig) error { +func StartComponents(cfg *config.Config) error { kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin) if err := startServiceCAController(cfg, kubeAdminConfig); err != nil { diff --git a/pkg/components/controllers.go b/pkg/components/controllers.go index 10ae6593c1..13c4038980 100644 --- a/pkg/components/controllers.go +++ b/pkg/components/controllers.go @@ -9,7 +9,7 @@ import ( "k8s.io/klog/v2" ) -func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startServiceCAController(cfg *config.Config, kubeconfigPath string) error { var ( //TODO: fix the rolebinding and sa clusterRoleBinding = []string{ @@ -101,7 +101,7 @@ func startServiceCAController(cfg *config.MicroshiftConfig, kubeconfigPath strin return nil } -func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startIngressController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-router/cluster-role-binding.yaml", @@ -178,7 +178,7 @@ func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string) return nil } -func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startDNSController(cfg *config.Config, kubeconfigPath string) error { var ( clusterRoleBinding = []string{ "components/openshift-dns/dns/cluster-role-binding.yaml", @@ -210,7 +210,7 @@ func startDNSController(cfg *config.MicroshiftConfig, kubeconfigPath string) err } extraParams := assets.RenderParams{ - "ClusterIP": cfg.Cluster.DNS, + "ClusterIP": cfg.Network.DNS, } if err := assets.ApplyServices(svc, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply service %v %v", svc, err) diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 0cefcd2e46..d0c75b2435 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -10,7 +10,7 @@ import ( "k8s.io/klog/v2" ) -func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCNIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/ovn/namespace.yaml", diff --git a/pkg/components/render.go b/pkg/components/render.go index b24723c5db..fd616d2fde 100755 --- a/pkg/components/render.go +++ b/pkg/components/render.go @@ -20,15 +20,15 @@ var templateFuncs = map[string]interface{}{ "Sha256sum": func(s string) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) }, } -func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderParams) assets.RenderParams { +func renderParamsFromConfig(cfg *config.Config, extra assets.RenderParams) assets.RenderParams { params := map[string]interface{}{ "ReleaseImage": release.Image, - "NodeName": cfg.NodeName, - "NodeIP": cfg.NodeIP, - "ClusterCIDR": cfg.Cluster.ClusterCIDR, - "ServiceCIDR": cfg.Cluster.ServiceCIDR, - "ClusterDNS": cfg.Cluster.DNS, - "BaseDomain": cfg.BaseDomain, + "NodeName": cfg.CanonicalNodeName(), + "NodeIP": cfg.Node.NodeIP, + "ClusterCIDR": cfg.Network.ClusterNetwork[0].CIDR, + "ServiceCIDR": cfg.Network.ServiceNetwork[0], + "ClusterDNS": cfg.Network.DNS, + "BaseDomain": cfg.DNS.BaseDomain, } for k, v := range extra { params[k] = v diff --git a/pkg/components/render_test.go b/pkg/components/render_test.go index 5498cfc65c..33b1e1fb88 100644 --- a/pkg/components/render_test.go +++ b/pkg/components/render_test.go @@ -101,7 +101,7 @@ func Test_renderTopolvmDaemonsetTemplate(t *testing.T) { name: "renders lvmd-socket-name path", args: args{ tb: tb, - data: renderParamsFromConfig(config.NewMicroshiftConfig(), assets.RenderParams{"SocketName": "/run/lvmd/lvmd.socket", "lvmd": "foobar"}), + data: renderParamsFromConfig(config.NewDefault(), assets.RenderParams{"SocketName": "/run/lvmd/lvmd.socket", "lvmd": "foobar"}), }, want: wantBytes(tpl, map[string]interface{}{ "ReleaseImage": release.Image, diff --git a/pkg/components/storage.go b/pkg/components/storage.go index 1d13590b1d..b4bd0b7531 100644 --- a/pkg/components/storage.go +++ b/pkg/components/storage.go @@ -24,7 +24,7 @@ func getCSIPluginConfig() (*lvmd.Lvmd, error) { return lvmd.DefaultLvmdConfig() } -func startCSIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error { +func startCSIPlugin(cfg *config.Config, kubeconfigPath string) error { var ( ns = []string{ "components/lvms/topolvm-openshift-storage_namespace.yaml", diff --git a/pkg/config/apiserver.go b/pkg/config/apiserver.go new file mode 100644 index 0000000000..88623d03e9 --- /dev/null +++ b/pkg/config/apiserver.go @@ -0,0 +1,19 @@ +package config + +type ApiServer struct { + // SubjectAltNames added to API server certs + SubjectAltNames []string `json:"subjectAltNames"` + // Kube apiserver advertise address to work around the certificates issue + // when requiring external access using the node IP. This will turn into + // the IP configured in the endpoint slice for kubernetes service. Must be + // a reachable IP from pods. Defaults to service network CIDR first + // address. + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + // Determines if kube-apiserver controller should configure the + // AdvertiseAddress in the loopback interface. Automatically computed. + SkipInterface bool `json:"-"` + + // The URL and Port of the API server cannot be changed by the user. + URL string `json:"-"` + Port int `json:"-"` +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 97f9847e1c..26e0c71b40 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,458 +2,191 @@ package config import ( "bytes" - "errors" "fmt" "net" "net/url" "os" "os/exec" - "path/filepath" - "strconv" "strings" "time" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/mitchellh/go-homedir" - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/component-base/logs" "k8s.io/klog/v2" - "sigs.k8s.io/yaml" + "github.com/apparentlymart/go-cidr/cidr" "github.com/openshift/microshift/pkg/util" ) const ( - DefaultUserConfigFile = "~/.microshift/config.yaml" - defaultUserDataDir = "~/.microshift/data" - DefaultGlobalConfigFile = "/etc/microshift/config.yaml" - defaultGlobalDataDir = "/var/lib/microshift" - // for files managed via management system in /etc, i.e. user applications - defaultManifestDirEtc = "/etc/microshift/manifests" - // for files embedded in ostree. i.e. cni/other component customizations - defaultManifestDirLib = "/usr/lib/microshift/manifests" // default DNS resolve file when systemd-resolved is used DefaultSystemdResolvedFile = "/run/systemd/resolve/resolv.conf" ) -var ( - configFile = findConfigFile() - dataDir = findDataDir() - manifestsDir = findManifestsDir() -) - -type ClusterConfig struct { - URL string `json:"-"` - ClusterCIDR string `json:"clusterCIDR"` - ServiceCIDR string `json:"serviceCIDR"` - ServiceNodePortRange string `json:"serviceNodePortRange"` - DNS string `json:"-"` -} - -type IngressConfig struct { - ServingCertificate []byte - ServingKey []byte -} - -type EtcdConfig struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimit uint64 - // The limit on the size of the etcd database; etcd will start failing writes if its size on disk reaches this value - QuotaBackendBytes int64 - // If the backend is fragmented more than `maxFragmentedPercentage` - // and the database size is greater than `minDefragBytes`, do a defrag. - MinDefragBytes int64 - MaxFragmentedPercentage float64 - // How often to check the conditions for defragging (0 means no defrags, except for a single on startup if `doStartupDefrag` is set). - DefragCheckFreq time.Duration - // Whether or not to do a defrag when the server finishes starting - DoStartupDefrag bool -} - -type MicroshiftConfig struct { - LogVLevel int `json:"logVLevel"` - - SubjectAltNames []string `json:"subjectAltNames"` - NodeName string `json:"nodeName"` - NodeIP string `json:"nodeIP"` - // Kube apiserver advertise address to work around the certificates issue - // when requiring external access using the node IP. This will turn into - // the IP configured in the endpoint slice for kubernetes service. Must be - // a reachable IP from pods. Defaults to service network CIDR first - // address. - KASAdvertiseAddress string `json:"kasAdvertiseAddress"` - // Determines if kube-apiserver controller should configure the - // KASAdvertiseAddress in the loopback interface. Automatically computed. - SkipKASInterface bool `json:"-"` - BaseDomain string `json:"baseDomain"` - Cluster ClusterConfig `json:"cluster"` - - Ingress IngressConfig `json:"-"` - Etcd EtcdConfig `json:"etcd"` -} - -// Top level config file type Config struct { - DNS DNS `json:"dns"` - Network Network `json:"network"` - Node Node `json:"node"` - ApiServer ApiServer `json:"apiServer"` - Debugging Debugging `json:"debugging"` - Etcd Etcd `json:"etcd"` -} - -const ( - // Etcd performance degrades significantly if the memory available is less than 50MB, enfore this minimum. - EtcdMinimumMemoryLimit = 50 -) - -type Etcd struct { - // Set a memory limit, in megabytes, on the etcd process; etcd will begin paging memory when it gets to this value. 0 means no limit. - MemoryLimitMB uint64 `json:"memoryLimitMB"` -} - -type Network struct { - // IP address pool to use for pod IPs. - // This field is immutable after installation. - ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork,omitempty"` - - // IP address pool for services. - // Currently, we only support a single entry here. - // This field is immutable after installation. - ServiceNetwork []string `json:"serviceNetwork,omitempty"` - - // The port range allowed for Services of type NodePort. - // If not specified, the default of 30000-32767 will be used. - // Such Services without a NodePort specified will have one - // automatically allocated from this range. - // This parameter can be updated after the cluster is - // installed. - // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` - ServiceNodePortRange string `json:"serviceNodePortRange,omitempty"` -} - -type ClusterNetworkEntry struct { - // The complete block for pod IPs. - CIDR string `json:"cidr,omitempty"` -} - -type DNS struct { - // baseDomain is the base domain of the cluster. All managed DNS records will - // be sub-domains of this base. - // - // For example, given the base domain `example.com`, router exposed - // domains will be formed as `*.apps.example.com` by default, - // and API service will have a DNS entry for `api.example.com`, - // as well as "api-int.example.com" for internal k8s API access. - // - // Once set, this field cannot be changed. - BaseDomain string `json:"baseDomain"` -} - -type ApiServer struct { - // SubjectAltNames added to API server certs - SubjectAltNames []string `json:"subjectAltNames"` - // AdvertiseAddress for endpoint slices in kubernetes service. Developer - // only parameter, wont show in show-config commands or docs. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` -} - -type Node struct { - // If non-empty, will use this string to identify the node instead of the hostname - HostnameOverride string `json:"hostnameOverride"` - - // IP address of the node, passed to the kubelet. - // If not specified, kubelet will use the node's default IP address. - NodeIP string `json:"nodeIP"` -} - -type Debugging struct { - // Valid values are: "Normal", "Debug", "Trace", "TraceAll". - // Defaults to "Normal". - LogLevel string `json:"logLevel"` -} - -func GetConfigFile() string { - return configFile -} - -func GetDataDir() string { - return dataDir -} + DNS DNS `json:"dns"` + Network Network `json:"network"` + Node Node `json:"node"` + ApiServer ApiServer `json:"apiServer"` + Etcd EtcdConfig `json:"etcd"` + Debugging Debugging `json:"debugging"` -func GetManifestsDir() []string { - return manifestsDir + // Internal-only fields + Ingress IngressConfig `json:"-"` + userSettings *Config `json:"-"` // the values read from the config file } -// KubeConfigID identifies the different kubeconfigs managed in the DataDir -type KubeConfigID string - -const ( - KubeAdmin KubeConfigID = "kubeadmin" - KubeControllerManager KubeConfigID = "kube-controller-manager" - KubeScheduler KubeConfigID = "kube-scheduler" - Kubelet KubeConfigID = "kubelet" - ClusterPolicyController KubeConfigID = "cluster-policy-controller" - RouteControllerManager KubeConfigID = "route-controller-manager" -) - -// KubeConfigPath returns the path to the specified kubeconfig file. -func (cfg *MicroshiftConfig) KubeConfigPath(id KubeConfigID) string { - return filepath.Join(dataDir, "resources", string(id), "kubeconfig") -} - -func (cfg *MicroshiftConfig) KubeConfigAdminPath(id string) string { - return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") -} - -var allHostnames []string - -func getAllHostnames() ([]string, error) { - if len(allHostnames) != 0 { - return allHostnames, nil +// NewDefault creates a new Config struct populated with the +// default values and with any computed values updated based on those +// defaults. +func NewDefault() *Config { + c := &Config{} + if err := c.fillDefaults(); err != nil { + klog.Fatalf("Failed to initialize config: %v", err) } - cmd := exec.Command("/bin/hostname", "-A") - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - return nil, fmt.Errorf("Error when executing 'hostname -A': %v", err) + if err := c.updateComputedValues(); err != nil { + klog.Fatalf("Failed to initialize config: %v", err) } - outString := out.String() - outString = strings.Trim(outString[:len(outString)-1], " ") - // Remove duplicates to avoid having them in the certificates. - names := strings.Split(outString, " ") - set := sets.NewString(names...) - allHostnames = set.List() - return allHostnames, nil + return c } -func NewMicroshiftConfig() *MicroshiftConfig { - nodeName, err := os.Hostname() +// fillDefaults forcibly sets the configuration to the default +// values. We do not use a static struct for the defaults because some +// of them are computed from the environment. If any error occurs +// probing the environment, the values in the Config instance are not +// changed. +func (c *Config) fillDefaults() error { + + // Look up any values that may generate an error + subjectAltNames, err := getAllHostnames() if err != nil { - klog.Fatalf("Failed to get hostname %v", err) + return fmt.Errorf("failed to get all hostnames: %v", err) } - nodeIP, err := util.GetHostIP() + hostname, err := os.Hostname() if err != nil { - klog.Fatalf("failed to get host IP: %v", err) + return fmt.Errorf("Failed to get hostname %v", err) } - subjectAltNames, err := getAllHostnames() + nodeIP, err := util.GetHostIP() if err != nil { - klog.Fatalf("failed to get all hostnames: %v", err) + return fmt.Errorf("failed to get host IP: %v", err) } - return &MicroshiftConfig{ - LogVLevel: 2, - SubjectAltNames: subjectAltNames, - NodeName: strings.ToLower(nodeName), - NodeIP: nodeIP, - BaseDomain: "example.com", - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.42.0.0/16", - ServiceCIDR: "10.43.0.0/16", - ServiceNodePortRange: "30000-32767", - }, - Etcd: EtcdConfig{ - MemoryLimit: 0, // No limit - MinDefragBytes: 100 * 1024 * 1024, // 100MB - MaxFragmentedPercentage: 45, // percent - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, // 8GB - }, + c.Debugging = Debugging{ + LogLevel: "Normal", } -} - -// Determine if the config file specified a NodeName (by default it's assigned the hostname) -func (c *MicroshiftConfig) isDefaultNodeName() bool { - hostname, err := os.Hostname() - if err != nil { - klog.Fatalf("Failed to get hostname %v", err) + c.ApiServer = ApiServer{ + SubjectAltNames: subjectAltNames, + URL: "https://localhost:6443", + Port: 6443, } - return c.NodeName == strings.ToLower(hostname) -} - -// Read or set the NodeName that will be used for this MicroShift instance -func (c *MicroshiftConfig) establishNodeName() (string, error) { - filePath := filepath.Join(GetDataDir(), ".nodename") - contents, err := os.ReadFile(filePath) - if os.IsNotExist(err) { - // ensure that dataDir exists - os.MkdirAll(GetDataDir(), 0700) - if err := os.WriteFile(filePath, []byte(c.NodeName), 0444); err != nil { - return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) - } - return c.NodeName, nil - } else if err != nil { - return "", err + c.Node = Node{ + HostnameOverride: hostname, + NodeIP: nodeIP, } - return string(contents), nil -} - -// Validate the NodeName to be used for this MicroShift instances -func (c *MicroshiftConfig) validateNodeName(isDefaultNodeName bool) error { - if addr := net.ParseIP(c.NodeName); addr != nil { - return fmt.Errorf("NodeName can not be an IP address: %q", c.NodeName) + c.DNS = DNS{ + BaseDomain: "example.com", } - - establishedNodeName, err := c.establishNodeName() - if err != nil { - return fmt.Errorf("failed to establish NodeName: %v", err) + c.Network = Network{ + ClusterNetwork: []ClusterNetworkEntry{ + { + CIDR: "10.42.0.0/16", + }, + }, + ServiceNetwork: []string{ + "10.43.0.0/16", + }, + ServiceNodePortRange: "30000-32767", + DNS: "10.43.0.10", } - - if establishedNodeName != c.NodeName { - if !isDefaultNodeName { - return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", - c.NodeName, establishedNodeName) - } else { - c.NodeName = establishedNodeName - klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ - "Please consider using a static NodeName in configuration", c.NodeName) - } + c.Etcd = EtcdConfig{ + MemoryLimitMB: 0, + QuotaBackendBytes: 8 * 1024 * 1024 * 1024, + MinDefragBytes: 100 * 1024 * 1024, + MaxFragmentedPercentage: 45, + DefragCheckFreq: 5 * time.Minute, } return nil } -// extract the api server port from the cluster URL -func (c *ClusterConfig) ApiServerPort() (int, error) { - var port string +// incorporateUserSettings merges any values read from the +// configuration file provided by the user with the existing settings +// (usually the defaults). +func (c *Config) incorporateUserSettings(u *Config) { + c.userSettings = u - parsed, err := url.Parse(c.URL) - if err != nil { - return 0, err + if u.DNS.BaseDomain != "" { + c.DNS.BaseDomain = u.DNS.BaseDomain } - // default empty URL to port 6443 - port = parsed.Port() - if port == "" { - port = "6443" - } - portNum, err := strconv.Atoi(port) - if err != nil { - return 0, err + if len(u.Network.ClusterNetwork) != 0 { + c.Network.ClusterNetwork = u.Network.ClusterNetwork } - return portNum, nil -} - -// Returns the default user config file if that exists, else the default global -// config file, else the empty string. -func findConfigFile() string { - userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) - if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { - if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { - return "" - } else { - return DefaultGlobalConfigFile + if len(u.Network.ServiceNetwork) != 0 { + c.Network.ServiceNetwork = u.Network.ServiceNetwork + // The default for the API server address is computed from the + // service network. If the user provides a network without + // also overriding the computed address, we need to clear the + // address here so it is recomputed later. If they provide + // both the network and the address, the address will be + // copied into place below with the other API server settings. + if u.ApiServer.AdvertiseAddress == "" { + c.ApiServer.AdvertiseAddress = "" } - } else { - return userConfigFile } -} - -// Returns the default user data dir if it exists or the user is non-root. -// Returns the default global data dir otherwise. -func findDataDir() string { - userDataDir, _ := homedir.Expand(defaultUserDataDir) - if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { - if os.Geteuid() > 0 { - return userDataDir - } else { - return defaultGlobalDataDir - } - } else { - return userDataDir + if u.Network.ServiceNodePortRange != "" { + c.Network.ServiceNodePortRange = u.Network.ServiceNodePortRange } -} - -// Returns the default manifests directories -func findManifestsDir() []string { - var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} - return manifestsDir -} - -func StringInList(s string, list []string) bool { - for _, x := range list { - if x == s { - return true - } + if u.Network.DNS != "" { + c.Network.DNS = u.Network.DNS } - return false -} -func (c *MicroshiftConfig) ReadFromConfigFile(configFile string) error { - contents, err := os.ReadFile(configFile) - if err != nil { - return fmt.Errorf("reading config file %q: %v", configFile, err) - } - var config Config - if err := yaml.Unmarshal(contents, &config); err != nil { - return fmt.Errorf("decoding config file %s: %v", configFile, err) + if u.Etcd.MemoryLimitMB != 0 { + c.Etcd.MemoryLimitMB = u.Etcd.MemoryLimitMB } - // Wire new Config type to existing MicroshiftConfig - c.LogVLevel = config.GetVerbosity() - if config.Node.HostnameOverride != "" { - c.NodeName = strings.ToLower(config.Node.HostnameOverride) + if u.Node.HostnameOverride != "" { + c.Node.HostnameOverride = u.Node.HostnameOverride } - if config.Node.NodeIP != "" { - c.NodeIP = config.Node.NodeIP + if u.Node.NodeIP != "" { + c.Node.NodeIP = u.Node.NodeIP } - if len(config.Network.ClusterNetwork) != 0 { - c.Cluster.ClusterCIDR = config.Network.ClusterNetwork[0].CIDR - } - if len(config.Network.ServiceNetwork) != 0 { - c.Cluster.ServiceCIDR = config.Network.ServiceNetwork[0] - } - if config.Network.ServiceNodePortRange != "" { - c.Cluster.ServiceNodePortRange = config.Network.ServiceNodePortRange - } - if config.DNS.BaseDomain != "" { - c.BaseDomain = config.DNS.BaseDomain + + if len(u.ApiServer.SubjectAltNames) != 0 { + c.ApiServer.SubjectAltNames = u.ApiServer.SubjectAltNames } - if len(config.ApiServer.SubjectAltNames) > 0 { - c.SubjectAltNames = config.ApiServer.SubjectAltNames + if u.ApiServer.AdvertiseAddress != "" { + c.ApiServer.AdvertiseAddress = u.ApiServer.AdvertiseAddress } - if len(config.ApiServer.AdvertiseAddress) > 0 { - c.KASAdvertiseAddress = config.ApiServer.AdvertiseAddress + if u.ApiServer.URL != "" { + c.ApiServer.URL = u.ApiServer.URL } - if config.Etcd.MemoryLimitMB > 0 { - // If the memory limit is than the minimum, set it to the minimum and continue. - if config.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { - c.Etcd.MemoryLimit = EtcdMinimumMemoryLimit - } else { - c.Etcd.MemoryLimit = config.Etcd.MemoryLimitMB - } + if u.Debugging.LogLevel != "" { + c.Debugging.LogLevel = u.Debugging.LogLevel } - - return nil } -// Note: add a configFile parameter here because of unit test requiring custom -// local directory -func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { - if configFile != "" { - if err := c.ReadFromConfigFile(configFile); err != nil { - return err - } - } +// updateComputedValues examins the existing settings and converts any +// inputs to more easily consumable units or fills in any defaults +// computed based on the values of other settings. +func (c *Config) updateComputedValues() error { - // validate serviceCIDR - clusterDNS, err := getClusterDNS(c.Cluster.ServiceCIDR) + clusterDNS, err := c.computeClusterDNS() if err != nil { - return fmt.Errorf("failed to get DNS IP: %v", err) + return err } - c.Cluster.DNS = clusterDNS + c.Network.DNS = clusterDNS - // If KAS advertise address is not configured then compute it from the service - // CIDR automatically. - if len(c.KASAdvertiseAddress) == 0 { + // If KAS advertise address configured, we do not want to apply + // the IP to the internal interface. + if c.userSettings != nil && len(c.userSettings.ApiServer.AdvertiseAddress) != 0 { + c.ApiServer.SkipInterface = true + } + + // If we have no advertise address, pick one. + if len(c.ApiServer.AdvertiseAddress) == 0 { // unchecked error because this was done when getting cluster DNS - _, svcNet, _ := net.ParseCIDR(c.Cluster.ServiceCIDR) + _, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0]) // Since the KAS advertise address was not provided we will default to the // next immediate subnet after the service CIDR. This is due to the fact // that using the actual apiserver service IP as an endpoint slice breaks @@ -466,13 +199,14 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { } // First and last are the same because of the /32 netmask. firstValidIP, _ := cidr.AddressRange(nextSubnet) - c.KASAdvertiseAddress = firstValidIP.String() - c.SkipKASInterface = false - } else { - c.SkipKASInterface = true + c.ApiServer.AdvertiseAddress = firstValidIP.String() } - if len(c.SubjectAltNames) > 0 { + return nil +} + +func (c *Config) validate() error { + if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. // Any of the hostnames and IPs (except the node IP) listed below conflicts with // other certificates, such as the service network and localhost access. @@ -485,25 +219,24 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { // the node IP it returns that certificate, which is the external access one. This // breaks all pods trying to reach apiserver, as hostnames dont match and the certificate // is invalid. - u, err := url.Parse(c.Cluster.URL) + u, err := url.Parse(c.ApiServer.URL) if err != nil { return fmt.Errorf("failed to parse cluster URL: %v", err) } if u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1" { - if stringSliceContains(c.SubjectAltNames, "localhost", "127.0.0.1") { + if stringSliceContains(c.ApiServer.SubjectAltNames, "localhost", "127.0.0.1") { return fmt.Errorf("subjectAltNames must not contain localhost, 127.0.0.1") } } else { - if stringSliceContains(c.SubjectAltNames, c.NodeIP) { + if stringSliceContains(c.ApiServer.SubjectAltNames, c.Node.NodeIP) { return fmt.Errorf("subjectAltNames must not contain node IP") } - if !stringSliceContains(c.SubjectAltNames, u.Host) || u.Host != c.NodeName { + if !stringSliceContains(c.ApiServer.SubjectAltNames, u.Host) || u.Host != c.Node.HostnameOverride { return fmt.Errorf("Cluster URL host %v is not included in subjectAltNames or nodeName", u.String()) } } - if stringSliceContains( - c.SubjectAltNames, + c.ApiServer.SubjectAltNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", @@ -512,76 +245,39 @@ func (c *MicroshiftConfig) ReadAndValidate(configFile string) error { "openshift.default", "openshift.default.svc", "openshift.default.svc.cluster.local", - c.KASAdvertiseAddress, + c.ApiServer.AdvertiseAddress, ) { return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs") } } - // Validate NodeName in config file, node-name should not be changed for an already - // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage - // being orphaned or lost, and other side effects. - if err := c.validateNodeName(c.isDefaultNodeName()); err != nil { - klog.Fatalf("Error in validating node name: %v", err) + + if c.Etcd.MemoryLimitMB > 0 && c.Etcd.MemoryLimitMB < EtcdMinimumMemoryLimit { + return fmt.Errorf("etcd.memoryLimitMB value %d is below the minimum allowed %d", + c.Etcd.MemoryLimitMB, EtcdMinimumMemoryLimit, + ) } return nil } -// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork -func getClusterDNS(serviceCIDR string) (string, error) { - _, service, err := net.ParseCIDR(serviceCIDR) - if err != nil { - return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) - } - dnsClusterIP, err := cidr.Host(service, 10) - if err != nil { - return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) - } - - return dnsClusterIP.String(), nil -} +var allHostnames []string -func stringSliceContains(list []string, elements ...string) bool { - for _, value := range list { - for _, element := range elements { - if value == element { - return true - } - } +func getAllHostnames() ([]string, error) { + if len(allHostnames) != 0 { + return allHostnames, nil } - return false -} - -// GetVerbosity returns the numerical value for LogLevel which is an enum -func (c *Config) GetVerbosity() int { - var verbosity int - switch c.Debugging.LogLevel { - case "Normal": - verbosity = 2 - case "Debug": - verbosity = 4 - case "Trace": - verbosity = 6 - case "TraceAll": - verbosity = 8 - default: - verbosity = 2 + cmd := exec.Command("/bin/hostname", "-A") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("Error when executing 'hostname -A': %v", err) } - return verbosity -} - -func HideUnsupportedFlags(flags *pflag.FlagSet) { - // hide logging flags that we do not use/support - loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) - logs.AddFlags(loggingFlags) - - supportedLoggingFlags := sets.NewString("v") - - loggingFlags.VisitAll(func(pf *pflag.Flag) { - if !supportedLoggingFlags.Has(pf.Name) { - flags.MarkHidden(pf.Name) - } - }) - - flags.MarkHidden("version") + outString := out.String() + outString = strings.Trim(outString[:len(outString)-1], " ") + // Remove duplicates to avoid having them in the certificates. + names := strings.Split(outString, " ") + set := sets.NewString(names...) + allHostnames = set.List() + return allHostnames, nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 3616c79a28..fc57668286 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,13 +1,13 @@ package config import ( + "fmt" "os" "path/filepath" - "reflect" + "strings" "testing" - "time" - "sigs.k8s.io/yaml" + "github.com/stretchr/testify/assert" ) const ( @@ -26,227 +26,306 @@ func setupSuiteDataDir(t *testing.T) func() { } } -func TestConfigFile(t *testing.T) { +// TestGetActiveConfigFromYAML verifies that reading the config file +// correctly overrides the defaults and updates the computed values in +// the Config struct. +func TestGetActiveConfigFromYAML(t *testing.T) { + mkDefaultConfig := func() *Config { + c := NewDefault() + return c + } + + dedent := func(input string) string { + lines := strings.Split(input, "\n") + detectIndentFrom := lines[0] + if detectIndentFrom == "" { + detectIndentFrom = lines[1] + } + dedentedLine := strings.TrimLeft(detectIndentFrom, " \t") + prefixLen := len(detectIndentFrom) - len(dedentedLine) + if prefixLen == 0 { + return input + } + var b strings.Builder + for _, line := range lines { + if len(line) >= prefixLen { + line = line[prefixLen:] + } + fmt.Fprintf(&b, "%s\n", line) + } + return b.String() + } + var ttests = []struct { - config Config - expected MicroshiftConfig + name string + config string + expected *Config expectErr bool }{ { - config: Config{ - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, + name: "empty", + config: "", + expected: mkDefaultConfig(), + }, + { + name: "dns", + config: dedent(` + dns: + baseDomain: test-example.com + `), + expected: func() *Config { + c := mkDefaultConfig() + c.DNS.BaseDomain = "test-example.com" + return c + }(), + }, + { + name: "network", + config: dedent(` + network: + clusterNetwork: + - cidr: "10.20.30.40/16" + serviceNetwork: + - "40.30.20.10/16" + serviceNodePortRange: "1024-32767" + `), + expected: func() *Config { + c := mkDefaultConfig() + c.Network.ClusterNetwork = []ClusterNetworkEntry{ + { + CIDR: "10.20.30.40/16", }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - }, - Debugging: Debugging{ - LogLevel: "Debug", - }, - }, - expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", - KASAdvertiseAddress: "6.7.8.9", - BaseDomain: "example.com", - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", - ServiceNodePortRange: "1024-32767", - }, - Etcd: EtcdConfig{ - MemoryLimit: 0, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, - MinDefragBytes: 100 * 1024 * 1024, - MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - }, - }, - expectErr: false, + } + c.Network.ServiceNetwork = []string{"40.30.20.10/16"} + c.Network.ServiceNodePortRange = "1024-32767" + c.ApiServer.AdvertiseAddress = "" // force value to be recomputed + c.updateComputedValues() // recomputes DNS field + return c + }(), + }, + { + name: "node", + config: dedent(` + node: + hostnameOverride: "node1" + nodeIP: "1.2.3.4" + `), + expected: func() *Config { + c := mkDefaultConfig() + c.Node.HostnameOverride = "node1" + c.Node.NodeIP = "1.2.3.4" + return c + }(), + }, + { + name: "api-server-subject-alt-names", + config: dedent(` + apiServer: + subjectAltNames: + - node1 + - node2 + `), + expected: func() *Config { + c := mkDefaultConfig() + c.ApiServer.SubjectAltNames = []string{ + "node1", "node2", + } + return c + }(), + }, + { + name: "api-server-advertise-address", + config: dedent(` + apiServer: + advertiseAddress: 4.3.2.1 + `), + expected: func() *Config { + c := mkDefaultConfig() + c.ApiServer.AdvertiseAddress = "4.3.2.1" + c.ApiServer.SkipInterface = true + return c + }(), + }, + { + name: "debugging", + config: dedent(` + debugging: + logLevel: Info + `), + expected: func() *Config { + c := mkDefaultConfig() + c.Debugging.LogLevel = "Info" + return c + }(), + }, + { + name: "etcd", + config: dedent(` + etcd: + memoryLimitMB: 100 + `), + expected: func() *Config { + c := mkDefaultConfig() + c.Etcd.MemoryLimitMB = 100 + c.updateComputedValues() + return c + }(), }, } + for _, tt := range ttests { - t.Run("", func(t *testing.T) { - f, err := os.CreateTemp("", "test") - if err != nil { - t.Errorf("unable to create temp file: %v", err) - } - defer os.Remove(f.Name()) - d, err := yaml.Marshal(&tt.config) - if err != nil { - t.Errorf("unable to marshal configuration: %v", err) - } - _, err = f.Write(d) - if err != nil { - t.Errorf("unable to write to file: %v", err) - } - config := NewMicroshiftConfig() - err = config.ReadFromConfigFile(f.Name()) + t.Run(tt.name, func(t *testing.T) { + + config, err := getActiveConfigFromYAML([]byte(tt.config)) + if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadFromConfigFile() mismatch. got=%v, want=%v", *config, tt.expected) + if !tt.expectErr { + + // blank out the user settings because the expected value + // never has them and any computed value should be set so + // it should be safe to ignore them + config.userSettings = nil + + assert.Equal(t, tt.expected, config, "config input:\n---%s\n---", tt.config) } }) } } -// test the MicroshiftConfig.ReadAndValidate function to verify that it configures MicroshiftConfig from -// a configuration file. -func TestMicroshiftConfigReadAndValidate(t *testing.T) { +// Test the validation logic +func TestValidate(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() + mkDefaultConfig := func() *Config { + c := NewDefault() + c.ApiServer.SkipInterface = true + return c + } + var ttests = []struct { name string - config Config - expected MicroshiftConfig + config *Config expectErr bool }{ { - name: "Config OK full", - config: Config{ - DNS: DNS{ - BaseDomain: "example.com", - }, - Network: Network{ - ClusterNetwork: []ClusterNetworkEntry{ - { - CIDR: "10.20.30.40/16", - }, - }, - ServiceNetwork: []string{"40.30.20.10/16"}, - ServiceNodePortRange: "1024-32767", - }, - Node: Node{ - HostnameOverride: "node1", - NodeIP: "1.2.3.4", - }, - ApiServer: ApiServer{ - SubjectAltNames: []string{"node1", "node2"}, - AdvertiseAddress: "6.7.8.9", - }, - Debugging: Debugging{ - LogLevel: "Debug", - }, - }, - expected: MicroshiftConfig{ - LogVLevel: 4, - SubjectAltNames: []string{"node1", "node2"}, - NodeName: "node1", - NodeIP: "1.2.3.4", - KASAdvertiseAddress: "6.7.8.9", - SkipKASInterface: true, - BaseDomain: "example.com", - Cluster: ClusterConfig{ - URL: "https://localhost:6443", - ClusterCIDR: "10.20.30.40/16", - ServiceCIDR: "40.30.20.10/16", - ServiceNodePortRange: "1024-32767", - DNS: "40.30.0.10", - }, - Etcd: EtcdConfig{ - MemoryLimit: 0, - QuotaBackendBytes: 8 * 1024 * 1024 * 1024, - MinDefragBytes: 100 * 1024 * 1024, - MaxFragmentedPercentage: 45, - DefragCheckFreq: 5 * time.Minute, - DoStartupDefrag: true, - }, - }, + name: "defaults-ok", + config: NewDefault(), expectErr: false, }, { - name: "Config NOK with bad SAN localhost", - config: Config{ - ApiServer: ApiServer{ - SubjectAltNames: []string{"127.0.0.1", "localhost"}, - }, - }, - expected: MicroshiftConfig{}, + name: "subject-alt-names-with-localhost", + config: func() *Config { + c := mkDefaultConfig() + c.ApiServer.SubjectAltNames = []string{"localhost"} + return c + }(), + expectErr: true, + }, + { + name: "subject-alt-names-with-loopback-ipv4", + config: func() *Config { + c := mkDefaultConfig() + c.ApiServer.SubjectAltNames = []string{"127.0.0.1"} + return c + }(), + expectErr: true, + }, + { + name: "subject-alt-names-with-kubernetes", + config: func() *Config { + c := mkDefaultConfig() + c.ApiServer.SubjectAltNames = []string{"kubernetes"} + return c + }(), expectErr: true, }, { - name: "Config NOK with bad SAN kubernetes service", - config: Config{ - ApiServer: ApiServer{ - SubjectAltNames: []string{"kubernetes"}, - }, - }, - expected: MicroshiftConfig{}, + name: "etcd-memory-limit-low", + config: func() *Config { + c := mkDefaultConfig() + c.Etcd.MemoryLimitMB = 1 + return c + }(), expectErr: true, }, + { + name: "etcd-memory-zero", + config: func() *Config { + c := mkDefaultConfig() + c.Etcd.MemoryLimitMB = 0 + return c + }(), + expectErr: false, + }, } for _, tt := range ttests { t.Run(tt.name, func(t *testing.T) { - f, err := os.CreateTemp("", "test") - if err != nil { - t.Errorf("unable to create temp file: %v", err) - } - defer os.Remove(f.Name()) - d, err := yaml.Marshal(&tt.config) - if err != nil { - t.Errorf("unable to marshal configuration: %v", err) - } - _, err = f.Write(d) - if err != nil { - t.Errorf("unable to write to file: %v", err) - } - config := NewMicroshiftConfig() - err = config.ReadAndValidate(f.Name()) + err := tt.config.validate() if tt.expectErr && err == nil { t.Fatal("Expecting error and received nothing") } if !tt.expectErr && err != nil { t.Fatalf("Not expecting error and received: %v", err) } - if !tt.expectErr && !reflect.DeepEqual(*config, tt.expected) { - t.Errorf("ReadAndValidate() mismatch. got=%v, want=%v", *config, tt.expected) - } }) } } func TestMicroshiftConfigIsDefaultNodeName(t *testing.T) { - c := NewMicroshiftConfig() + c := NewDefault() if !c.isDefaultNodeName() { t.Errorf("expected default IsDefaultNodeName to be true") } - c.NodeName += "-suffix" + c.Node.HostnameOverride += "-suffix" if c.isDefaultNodeName() { t.Errorf("expected default IsDefaultNodeName to be false") } } +func TestCanonicalNodeName(t *testing.T) { + hostname, _ := os.Hostname() + + var ttests = []struct { + name string + value string + expected string + }{ + { + name: "default", + value: "", + expected: strings.ToLower(hostname), + }, + { + name: "upper-case", + value: "Hostname", + expected: "hostname", + }, + } + + for _, tt := range ttests { + t.Run(tt.name, func(t *testing.T) { + c := NewDefault() + if tt.value != "" { // account for default + c.Node.HostnameOverride = tt.value + } + assert.Equal(t, tt.expected, c.CanonicalNodeName()) + }) + } +} + func TestMicroshiftConfigNodeNameValidation(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - c := NewMicroshiftConfig() - c.NodeName = "node1" + c := NewDefault() + c.Node.HostnameOverride = "node1" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err != nil { t.Errorf("failed to validate node name on first call: %v", err) @@ -255,7 +334,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { nodeNameFile := filepath.Join(dataDir, ".nodename") if data, err := os.ReadFile(nodeNameFile); err != nil { t.Errorf("failed to read node name from file %q: %v", nodeNameFile, err) - } else if string(data) != c.NodeName { + } else if string(data) != c.Node.HostnameOverride { t.Errorf("node name file doesn't match the node name in the saved file: %v", err) } @@ -263,7 +342,7 @@ func TestMicroshiftConfigNodeNameValidation(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_NOT_DEFAULT_NODENAME); err == nil { t.Errorf("validation should have failed for nodename change: %v", err) } @@ -273,7 +352,7 @@ func TestMicroshiftConfigNodeNameValidationFromDefault(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - c := NewMicroshiftConfig() + c := NewDefault() if err := c.validateNodeName(IS_DEFAULT_NODENAME); err != nil { t.Errorf("failed to validate node name on first call: %v", err) @@ -291,7 +370,7 @@ func TestMicroshiftConfigNodeNameValidationFromDefault(t *testing.T) { t.Errorf("failed to validate node name on second call without changes: %v", err) } - c.NodeName = "node2" + c.Node.HostnameOverride = "node2" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err != nil { t.Errorf("validation should have failed in this case, it must be a warning in logs: %v", err) } @@ -301,8 +380,8 @@ func TestMicroshiftConfigNodeNameValidationBadName(t *testing.T) { cleanup := setupSuiteDataDir(t) defer cleanup() - c := NewMicroshiftConfig() - c.NodeName = "1.2.3.4" + c := NewDefault() + c.Node.HostnameOverride = "1.2.3.4" if err := c.validateNodeName(IS_DEFAULT_NODENAME); err == nil { t.Errorf("failed to validate node name.") diff --git a/pkg/config/debugging.go b/pkg/config/debugging.go new file mode 100644 index 0000000000..a540f5490c --- /dev/null +++ b/pkg/config/debugging.go @@ -0,0 +1,25 @@ +package config + +type Debugging struct { + // Valid values are: "Normal", "Debug", "Trace", "TraceAll". + // Defaults to "Normal". + LogLevel string `json:"logLevel"` +} + +// GetVerbosity returns the numerical value for LogLevel which is an enum +func (c *Config) GetVerbosity() int { + var verbosity int + switch c.Debugging.LogLevel { + case "Normal": + verbosity = 2 + case "Debug": + verbosity = 4 + case "Trace": + verbosity = 6 + case "TraceAll": + verbosity = 8 + default: + verbosity = 2 + } + return verbosity +} diff --git a/pkg/config/debugging_test.go b/pkg/config/debugging_test.go new file mode 100644 index 0000000000..4ac79df68b --- /dev/null +++ b/pkg/config/debugging_test.go @@ -0,0 +1,48 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetVerbosity(t *testing.T) { + var ttests = []struct { + setting string + level int + }{ + { + setting: "Normal", + level: 2, + }, + { + setting: "Debug", + level: 4, + }, + { + setting: "Trace", + level: 6, + }, + { + setting: "TraceAll", + level: 8, + }, + { + setting: "Unknown", + level: 2, + }, + { + setting: "", + level: 2, + }, + } + + for _, tt := range ttests { + t.Run(tt.setting, func(t *testing.T) { + config := NewDefault() + config.Debugging.LogLevel = tt.setting + verbosity := config.GetVerbosity() + assert.Equal(t, tt.level, verbosity) + }) + } +} diff --git a/pkg/config/dns.go b/pkg/config/dns.go new file mode 100644 index 0000000000..ca6f11e88e --- /dev/null +++ b/pkg/config/dns.go @@ -0,0 +1,14 @@ +package config + +type DNS struct { + // baseDomain is the base domain of the cluster. All managed DNS records will + // be sub-domains of this base. + // + // For example, given the base domain `example.com`, router exposed + // domains will be formed as `*.apps.example.com` by default, + // and API service will have a DNS entry for `api.example.com`, + // as well as "api-int.example.com" for internal k8s API access. + // + // Once set, this field cannot be changed. + BaseDomain string `json:"baseDomain"` +} diff --git a/pkg/config/etcd.go b/pkg/config/etcd.go new file mode 100644 index 0000000000..cb571eb963 --- /dev/null +++ b/pkg/config/etcd.go @@ -0,0 +1,29 @@ +package config + +import "time" + +const ( + // Etcd performance degrades significantly if the memory available + // is less than 50MB, enforce this minimum. + EtcdMinimumMemoryLimit = 50 +) + +type EtcdConfig struct { + // Set a memory limit on the etcd process; etcd will begin paging + // memory when it gets to this value. 0 means no limit. + MemoryLimitMB uint64 `json:"memoryLimitMB"` + + // The limit on the size of the etcd database; etcd will start + // failing writes if its size on disk reaches this value + QuotaBackendBytes int64 `json:"-"` + + // If the backend is fragmented more than + // `maxFragmentedPercentage` and the database size is greater than + // `minDefragBytes`, do a defrag. + MinDefragBytes int64 `json:"-"` + MaxFragmentedPercentage float64 `json:"-"` + + // How often to check the conditions for defragging (0 means no + // defrags, except for a single on startup). + DefragCheckFreq time.Duration `json:"-"` +} diff --git a/pkg/config/files.go b/pkg/config/files.go new file mode 100644 index 0000000000..6c78397125 --- /dev/null +++ b/pkg/config/files.go @@ -0,0 +1,126 @@ +package config + +import ( + "errors" + "fmt" + "os" + + "github.com/mitchellh/go-homedir" + "sigs.k8s.io/yaml" +) + +const ( + DefaultUserConfigFile = "~/.microshift/config.yaml" + defaultUserDataDir = "~/.microshift/data" + DefaultGlobalConfigFile = "/etc/microshift/config.yaml" + defaultGlobalDataDir = "/var/lib/microshift" + // for files managed via management system in /etc, i.e. user applications + defaultManifestDirEtc = "/etc/microshift/manifests" + // for files embedded in ostree. i.e. cni/other component customizations + defaultManifestDirLib = "/usr/lib/microshift/manifests" +) + +var ( + configFile = findConfigFile() + dataDir = findDataDir() + manifestsDir = findManifestsDir() +) + +func GetConfigFile() string { + return configFile +} + +func GetDataDir() string { + return dataDir +} + +func GetManifestsDir() []string { + return manifestsDir +} + +// Returns the default user config file if that exists, else the default global +// config file, else the empty string. +func findConfigFile() string { + userConfigFile, _ := homedir.Expand(DefaultUserConfigFile) + if _, err := os.Stat(userConfigFile); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(DefaultGlobalConfigFile); errors.Is(err, os.ErrNotExist) { + return "" + } else { + return DefaultGlobalConfigFile + } + } else { + return userConfigFile + } +} + +// Returns the default user data dir if it exists or the user is non-root. +// Returns the default global data dir otherwise. +func findDataDir() string { + userDataDir, _ := homedir.Expand(defaultUserDataDir) + if _, err := os.Stat(userDataDir); errors.Is(err, os.ErrNotExist) { + if os.Geteuid() > 0 { + return userDataDir + } else { + return defaultGlobalDataDir + } + } else { + return userDataDir + } +} + +// Returns the default manifests directories +func findManifestsDir() []string { + var manifestsDir = []string{defaultManifestDirLib, defaultManifestDirEtc} + return manifestsDir +} + +func parse(contents []byte) (*Config, error) { + c := &Config{} + if err := yaml.Unmarshal(contents, c); err != nil { + return nil, fmt.Errorf("Unable to decode configuration: %v", err) + } + return c, nil +} + +func getActiveConfigFromYAML(contents []byte) (*Config, error) { + userSettings, err := parse(contents) + if err != nil { + return nil, fmt.Errorf("Error parsing config file %q: %v", configFile, err) + } + + // Start with the defaults, then apply the user settings and + // recompute dynamic values. + results := &Config{} + if err := results.fillDefaults(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + results.incorporateUserSettings(userSettings) + if err := results.updateComputedValues(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + if err := results.validate(); err != nil { + return nil, fmt.Errorf("Invalid configuration: %v", err) + } + return results, nil +} + +// ActiveConfig returns the active configuration. If the configuration +// file exists, read it and require it to be valid. Otherwise return +// the default settings. +func ActiveConfig() (*Config, error) { + filename := GetConfigFile() + _, err := os.Stat(filename) + if os.IsNotExist(err) { + // No configuration file, use the default settings + return NewDefault(), nil + } else if err != nil { + return nil, err + } + + // Read the file and merge user-provided settings with the defaults + contents, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("Error reading config file %q: %v", configFile, err) + } + return getActiveConfigFromYAML(contents) +} diff --git a/pkg/config/flags.go b/pkg/config/flags.go new file mode 100644 index 0000000000..e15ee75a1c --- /dev/null +++ b/pkg/config/flags.go @@ -0,0 +1,23 @@ +package config + +import ( + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/logs" +) + +func HideUnsupportedFlags(flags *pflag.FlagSet) { + // hide logging flags that we do not use/support + loggingFlags := pflag.NewFlagSet("logging-flags", pflag.ContinueOnError) + logs.AddFlags(loggingFlags) + + supportedLoggingFlags := sets.NewString("v") + + loggingFlags.VisitAll(func(pf *pflag.Flag) { + if !supportedLoggingFlags.Has(pf.Name) { + flags.MarkHidden(pf.Name) + } + }) + + flags.MarkHidden("version") +} diff --git a/pkg/config/ingress.go b/pkg/config/ingress.go new file mode 100644 index 0000000000..a8980ae1ed --- /dev/null +++ b/pkg/config/ingress.go @@ -0,0 +1,6 @@ +package config + +type IngressConfig struct { + ServingCertificate []byte + ServingKey []byte +} diff --git a/pkg/config/kubeconfig.go b/pkg/config/kubeconfig.go new file mode 100644 index 0000000000..1e33b07b96 --- /dev/null +++ b/pkg/config/kubeconfig.go @@ -0,0 +1,24 @@ +package config + +import "path/filepath" + +// KubeConfigID identifies the different kubeconfigs managed in the DataDir +type KubeConfigID string + +const ( + KubeAdmin KubeConfigID = "kubeadmin" + KubeControllerManager KubeConfigID = "kube-controller-manager" + KubeScheduler KubeConfigID = "kube-scheduler" + Kubelet KubeConfigID = "kubelet" + ClusterPolicyController KubeConfigID = "cluster-policy-controller" + RouteControllerManager KubeConfigID = "route-controller-manager" +) + +// KubeConfigPath returns the path to the specified kubeconfig file. +func (cfg *Config) KubeConfigPath(id KubeConfigID) string { + return filepath.Join(dataDir, "resources", string(id), "kubeconfig") +} + +func (cfg *Config) KubeConfigAdminPath(id string) string { + return filepath.Join(dataDir, "resources", string(KubeAdmin), id, "kubeconfig") +} diff --git a/pkg/config/network.go b/pkg/config/network.go new file mode 100644 index 0000000000..3a3f8cc211 --- /dev/null +++ b/pkg/config/network.go @@ -0,0 +1,62 @@ +package config + +import ( + "fmt" + "net" + + "github.com/apparentlymart/go-cidr/cidr" +) + +type Network struct { + // IP address pool to use for pod IPs. + // This field is immutable after installation. + ClusterNetwork []ClusterNetworkEntry `json:"clusterNetwork"` + + // IP address pool for services. + // Currently, we only support a single entry here. + // This field is immutable after installation. + ServiceNetwork []string `json:"serviceNetwork"` + + // The port range allowed for Services of type NodePort. + // If not specified, the default of 30000-32767 will be used. + // Such Services without a NodePort specified will have one + // automatically allocated from this range. + // This parameter can be updated after the cluster is + // installed. + // +kubebuilder:validation:Pattern=`^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])-([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$` + ServiceNodePortRange string `json:"serviceNodePortRange"` + + // The DNS server to use + DNS string `json:"-"` +} + +type ClusterNetworkEntry struct { + // The complete block for pod IPs. + CIDR string `json:"cidr"` +} + +func (c *Config) computeClusterDNS() (string, error) { + if len(c.Network.ServiceNetwork) == 0 { + return "", fmt.Errorf("network.serviceNetwork not filled in") + } + + clusterDNS, err := getClusterDNS(c.Network.ServiceNetwork[0]) + if err != nil { + return "", fmt.Errorf("failed to get DNS IP: %v", err) + } + return clusterDNS, nil +} + +// getClusterDNS returns cluster DNS IP that is 10th IP of the ServiceNetwork +func getClusterDNS(serviceCIDR string) (string, error) { + _, service, err := net.ParseCIDR(serviceCIDR) + if err != nil { + return "", fmt.Errorf("invalid service cidr %v: %v", serviceCIDR, err) + } + dnsClusterIP, err := cidr.Host(service, 10) + if err != nil { + return "", fmt.Errorf("service cidr must have at least 10 distinct host addresses %v: %v", serviceCIDR, err) + } + + return dnsClusterIP.String(), nil +} diff --git a/pkg/config/node.go b/pkg/config/node.go new file mode 100644 index 0000000000..13eb6e6a90 --- /dev/null +++ b/pkg/config/node.go @@ -0,0 +1,87 @@ +package config + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "k8s.io/klog/v2" +) + +type Node struct { + // If non-empty, will use this string to identify the node instead of the hostname + HostnameOverride string `json:"hostnameOverride"` + + // IP address of the node, passed to the kubelet. + // If not specified, kubelet will use the node's default IP address. + NodeIP string `json:"nodeIP"` +} + +// Determine if the config file specified a NodeName (by default it's assigned the hostname) +func (c *Config) isDefaultNodeName() bool { + hostname, err := os.Hostname() + if err != nil { + klog.Fatalf("Failed to get hostname %v", err) + } + return c.CanonicalNodeName() == strings.ToLower(hostname) +} + +// CanonicalNodeName returns the name to use for the node. The value +// is taken from either the HostnameOverride provided by the user in +// the config file, or the host name. +func (c *Config) CanonicalNodeName() string { + return strings.ToLower(c.Node.HostnameOverride) +} + +// Read or set the NodeName that will be used for this MicroShift instance +func (c *Config) establishNodeName() (string, error) { + name := c.CanonicalNodeName() + filePath := filepath.Join(GetDataDir(), ".nodename") + contents, err := os.ReadFile(filePath) + if os.IsNotExist(err) { + // ensure that dataDir exists + os.MkdirAll(GetDataDir(), 0700) + if err := os.WriteFile(filePath, []byte(name), 0444); err != nil { + return "", fmt.Errorf("failed to write nodename file %q: %v", filePath, err) + } + return name, nil + } else if err != nil { + return "", err + } + return string(contents), nil +} + +// Validate the NodeName to be used for this MicroShift instances +func (c *Config) validateNodeName(isDefaultNodeName bool) error { + currentNodeName := c.CanonicalNodeName() + if addr := net.ParseIP(currentNodeName); addr != nil { + return fmt.Errorf("NodeName can not be an IP address: %q", currentNodeName) + } + + establishedNodeName, err := c.establishNodeName() + if err != nil { + return fmt.Errorf("failed to establish NodeName: %v", err) + } + + if establishedNodeName != currentNodeName { + if !isDefaultNodeName { + return fmt.Errorf("configured NodeName %q does not match previous NodeName %q , NodeName cannot be changed for a device once established", + currentNodeName, establishedNodeName) + } else { + c.Node.HostnameOverride = establishedNodeName + klog.Warningf("NodeName has changed due to a host name change, using previously established NodeName %q."+ + "Please consider using a static NodeName in configuration", establishedNodeName) + } + } + + return nil +} + +func (c *Config) EnsureNodeNameHasNotChanged() error { + // Validate NodeName in config file, node-name should not be changed for an already + // initialized MicroShift instance. This can lead to Pods being re-scheduled, storage + // being orphaned or lost, and other side effects. + return c.validateNodeName(c.isDefaultNodeName()) +} diff --git a/pkg/config/util.go b/pkg/config/util.go new file mode 100644 index 0000000000..6bb69a5079 --- /dev/null +++ b/pkg/config/util.go @@ -0,0 +1,21 @@ +package config + +func StringInList(s string, list []string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func stringSliceContains(list []string, elements ...string) bool { + for _, value := range list { + for _, element := range elements { + if value == element { + return true + } + } + } + return false +} diff --git a/pkg/controllers/cluster-policy-controller.go b/pkg/controllers/cluster-policy-controller.go index d94fc54de0..45ae5e9f7f 100644 --- a/pkg/controllers/cluster-policy-controller.go +++ b/pkg/controllers/cluster-policy-controller.go @@ -35,7 +35,7 @@ type ClusterPolicyController struct { configErr error } -func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyController { +func NewClusterPolicyController(cfg *config.Config) *ClusterPolicyController { s := &ClusterPolicyController{} s.configErr = s.configure(cfg) return s @@ -44,7 +44,7 @@ func NewClusterPolicyController(cfg *config.MicroshiftConfig) *ClusterPolicyCont func (s *ClusterPolicyController) Name() string { return "cluster-policy-controller" } func (s *ClusterPolicyController) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *ClusterPolicyController) configure(cfg *config.MicroshiftConfig) error { +func (s *ClusterPolicyController) configure(cfg *config.Config) error { s.kubeconfig = cfg.KubeConfigPath(config.ClusterPolicyController) scheme := runtime.NewScheme() diff --git a/pkg/controllers/etcd.go b/pkg/controllers/etcd.go index 98e56458ae..d875652207 100644 --- a/pkg/controllers/etcd.go +++ b/pkg/controllers/etcd.go @@ -40,9 +40,9 @@ type EtcdService struct { memoryLimit uint64 } -func NewEtcd(cfg *config.MicroshiftConfig) *EtcdService { +func NewEtcd(cfg *config.Config) *EtcdService { return &EtcdService{ - memoryLimit: cfg.Etcd.MemoryLimit, + memoryLimit: cfg.Etcd.MemoryLimitMB, } } @@ -61,13 +61,13 @@ func (s *EtcdService) Run(ctx context.Context, ready chan<- struct{}, stopped ch return fmt.Errorf("%v failed to get exec path: %v", s.Name(), err) } etcdPath := filepath.Join(filepath.Dir(microshiftExecPath), "microshift-etcd") - // Not running the etcd binary directly, the proper etcd arguments are handled - // in etcd/cmd/microshift-etcd/run.go. + // Not running the etcd binary directly, the proper etcd arguments + // are handled in etcd/cmd/microshift-etcd/run.go. args := []string{} - // If we're launching MicroShift as a service, we need to do the same - // with etcd, so wrap it in a transient systemd-unit that's tied - // to the MicroShift service lifetime. + // If we're launching MicroShift as a service, we need to do the + // same with etcd, so wrap it in a transient systemd-unit that's + // tied to the MicroShift service lifetime. var exe string if runningAsSvc { args = append(args, @@ -89,6 +89,7 @@ func (s *EtcdService) Run(ctx context.Context, ready chan<- struct{}, stopped ch } args = append(args, "run") // Not using context as canceling ctx sends SIGKILL to process + klog.Infof("starting etcd via %s with args %v", exe, args) cmd := exec.Command(exe, args...) wd, err := os.Getwd() diff --git a/pkg/controllers/infra-services-controller.go b/pkg/controllers/infra-services-controller.go index 08a5c85298..7c5f8af766 100644 --- a/pkg/controllers/infra-services-controller.go +++ b/pkg/controllers/infra-services-controller.go @@ -26,10 +26,10 @@ import ( ) type InfrastructureServicesManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewInfrastructureServices(cfg *config.MicroshiftConfig) *InfrastructureServicesManager { +func NewInfrastructureServices(cfg *config.Config) *InfrastructureServicesManager { s := &InfrastructureServicesManager{} s.cfg = cfg return s @@ -62,7 +62,7 @@ func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- st return ctx.Err() } -func applyDefaultRBACs(cfg *config.MicroshiftConfig) error { +func applyDefaultRBACs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( cr = []string{ diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go index 49c0edbae4..96fbcead51 100644 --- a/pkg/controllers/kube-apiserver.go +++ b/pkg/controllers/kube-apiserver.go @@ -80,7 +80,7 @@ type KubeAPIServer struct { advertiseAddress string } -func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { +func NewKubeAPIServer(cfg *config.Config) *KubeAPIServer { s := &KubeAPIServer{} if err := s.configure(cfg); err != nil { s.configureErr = err @@ -91,8 +91,8 @@ func NewKubeAPIServer(cfg *config.MicroshiftConfig) *KubeAPIServer { func (s *KubeAPIServer) Name() string { return "kube-apiserver" } func (s *KubeAPIServer) Dependencies() []string { return []string{"etcd", "network-configuration"} } -func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { - s.verbosity = cfg.LogVLevel +func (s *KubeAPIServer) configure(cfg *config.Config) error { + s.verbosity = cfg.GetVerbosity() certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) kubeCSRSignerDir := cryptomaterial.CSRSignerCertDir(certsDir) @@ -109,15 +109,9 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { return fmt.Errorf("failed to configure kube-apiserver audit policy: %w", err) } - // Get the apiserver port so we can set it as an argument - apiServerPort, err := cfg.Cluster.ApiServerPort() - if err != nil { - return err - } - - s.masterURL = cfg.Cluster.URL + s.masterURL = cfg.ApiServer.URL s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir) - s.advertiseAddress = cfg.KASAdvertiseAddress + s.advertiseAddress = cfg.ApiServer.AdvertiseAddress overrides := &kubecontrolplanev1.KubeAPIServerConfig{ APIServerArguments: map[string]kubecontrolplanev1.Arguments{ @@ -139,7 +133,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { "proxy-client-key-file": {cryptomaterial.ClientKeyPath(aggregatorClientCertDir)}, "requestheader-client-ca-file": {aggregatorCAPath}, "service-account-signing-key-file": {microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key"}, - "service-node-port-range": {cfg.Cluster.ServiceNodePortRange}, + "service-node-port-range": {cfg.Network.ServiceNodePortRange}, "tls-cert-file": {servingCert}, "tls-private-key-file": {servingKey}, "disable-admission-plugins": { @@ -172,7 +166,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { APIVersion: "route.openshift.io/v1", Kind: "HostAssignmentAdmissionConfig", }, - Domain: "apps." + cfg.BaseDomain, + Domain: "apps." + cfg.DNS.BaseDomain, }, }, }, @@ -185,7 +179,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { }, ServingInfo: configv1.HTTPServingInfo{ ServingInfo: configv1.ServingInfo{ - BindAddress: net.JoinHostPort("0.0.0.0", strconv.Itoa(apiServerPort)), + BindAddress: net.JoinHostPort("0.0.0.0", strconv.Itoa(cfg.ApiServer.Port)), MinTLSVersion: string(fixedTLSProfile.MinTLSVersion), CipherSuites: crypto.OpenSSLToIANACipherSuites(fixedTLSProfile.Ciphers), NamedCertificates: []configv1.NamedCertificate{ @@ -214,8 +208,8 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { ServiceAccountPublicKeyFiles: []string{ microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.pub", }, - ServicesSubnet: cfg.Cluster.ServiceCIDR, - ServicesNodePortRange: cfg.Cluster.ServiceNodePortRange, + ServicesSubnet: cfg.Network.ServiceNetwork[0], + ServicesNodePortRange: cfg.Network.ServiceNodePortRange, } overridesBytes, err := json.Marshal(overrides) @@ -255,7 +249,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error { return nil } -func (s *KubeAPIServer) configureAuditPolicy(cfg *config.MicroshiftConfig) error { +func (s *KubeAPIServer) configureAuditPolicy(cfg *config.Config) error { data := []byte(` apiVersion: audit.k8s.io/v1 kind: Policy diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go index a9e6409e04..3a2126cb58 100644 --- a/pkg/controllers/kube-controller-manager.go +++ b/pkg/controllers/kube-controller-manager.go @@ -49,7 +49,7 @@ type KubeControllerManager struct { configureErr error } -func NewKubeControllerManager(cfg *config.MicroshiftConfig) *KubeControllerManager { +func NewKubeControllerManager(cfg *config.Config) *KubeControllerManager { s := &KubeControllerManager{} // TODO: manage and invoke the configure bits independently outside of this. s.args, s.applyFn, s.configureErr = configure(cfg) @@ -74,7 +74,7 @@ func kcmServiceAccountPrivateKeyFile() string { return microshiftDataDir + "/resources/kube-apiserver/secrets/service-account-key/service-account.key" } -func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() error, err error) { +func configure(cfg *config.Config) (args []string, applyFn func() error, err error) { kubeConfig := cfg.KubeConfigPath(config.KubeControllerManager) clusterSigningKey, clusterSigningCert := kcmClusterSigningCertKeyAndFile() @@ -85,7 +85,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "authorization-kubeconfig": {kubeConfig}, "service-account-private-key-file": {kcmServiceAccountPrivateKeyFile()}, "allocate-node-cidrs": {"true"}, - "cluster-cidr": {cfg.Cluster.ClusterCIDR}, + "cluster-cidr": {cfg.Network.ClusterNetwork[0].CIDR}, "root-ca-file": {kcmRootCAFile()}, "bind-address": {"127.0.0.1"}, "secure-port": {"10257"}, @@ -93,7 +93,7 @@ func configure(cfg *config.MicroshiftConfig) (args []string, applyFn func() erro "use-service-account-credentials": {"true"}, "cluster-signing-cert-file": {clusterSigningCert}, "cluster-signing-key-file": {clusterSigningKey}, - "v": {strconv.Itoa(cfg.LogVLevel)}, + "v": {strconv.Itoa(cfg.GetVerbosity())}, }, } diff --git a/pkg/controllers/kube-controller-manager_test.go b/pkg/controllers/kube-controller-manager_test.go index da2c9aae2f..110e31e486 100644 --- a/pkg/controllers/kube-controller-manager_test.go +++ b/pkg/controllers/kube-controller-manager_test.go @@ -37,7 +37,7 @@ func TestKCMDefaultConfigAsset(t *testing.T) { } func TestConfigure(t *testing.T) { - cfg := config.NewMicroshiftConfig() + cfg := config.NewDefault() kcm := NewKubeControllerManager(cfg) clusterSigningKey, clusterSigningCert := kcmClusterSigningCertKeyAndFile() diff --git a/pkg/controllers/kube-scheduler.go b/pkg/controllers/kube-scheduler.go index 78b219a259..d1cd500799 100644 --- a/pkg/controllers/kube-scheduler.go +++ b/pkg/controllers/kube-scheduler.go @@ -39,7 +39,7 @@ type KubeScheduler struct { kubeconfig string } -func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { +func NewKubeScheduler(cfg *config.Config) *KubeScheduler { s := &KubeScheduler{} s.configure(cfg) return s @@ -48,7 +48,7 @@ func NewKubeScheduler(cfg *config.MicroshiftConfig) *KubeScheduler { func (s *KubeScheduler) Name() string { return "kube-scheduler" } func (s *KubeScheduler) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { +func (s *KubeScheduler) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("failed to write kube-scheduler config: %v", err) } @@ -60,7 +60,7 @@ func (s *KubeScheduler) configure(cfg *config.MicroshiftConfig) { s.kubeconfig = cfg.KubeConfigPath(config.KubeScheduler) } -func (s *KubeScheduler) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeScheduler) writeConfig(cfg *config.Config) error { data := []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration clientConnection: diff --git a/pkg/controllers/openshift-crd-manager.go b/pkg/controllers/openshift-crd-manager.go index f505b650d0..de328afc89 100644 --- a/pkg/controllers/openshift-crd-manager.go +++ b/pkg/controllers/openshift-crd-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftCRDManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftCRDManager(cfg *config.MicroshiftConfig) *OpenShiftCRDManager { +func NewOpenShiftCRDManager(cfg *config.Config) *OpenShiftCRDManager { s := &OpenShiftCRDManager{} s.cfg = cfg return s diff --git a/pkg/controllers/openshift-default-scc-manager.go b/pkg/controllers/openshift-default-scc-manager.go index d933f2015a..19615724c1 100644 --- a/pkg/controllers/openshift-default-scc-manager.go +++ b/pkg/controllers/openshift-default-scc-manager.go @@ -24,10 +24,10 @@ import ( ) type OpenShiftDefaultSCCManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewOpenShiftDefaultSCCManager(cfg *config.MicroshiftConfig) *OpenShiftDefaultSCCManager { +func NewOpenShiftDefaultSCCManager(cfg *config.Config) *OpenShiftDefaultSCCManager { s := &OpenShiftDefaultSCCManager{} s.cfg = cfg return s @@ -51,7 +51,7 @@ func (s *OpenShiftDefaultSCCManager) Run(ctx context.Context, ready chan<- struc return ctx.Err() } -func ApplyDefaultSCCs(cfg *config.MicroshiftConfig) error { +func ApplyDefaultSCCs(cfg *config.Config) error { kubeconfigPath := cfg.KubeConfigPath(config.KubeAdmin) var ( clusterRole = []string{ diff --git a/pkg/controllers/openshift-route-controller-manager.go b/pkg/controllers/openshift-route-controller-manager.go index bf7df109f0..3bfa301fab 100644 --- a/pkg/controllers/openshift-route-controller-manager.go +++ b/pkg/controllers/openshift-route-controller-manager.go @@ -44,7 +44,7 @@ const ( componentRCM = "route-controller-manager" ) -func NewRouteControllerManager(cfg *config.MicroshiftConfig) *OCPRouteControllerManager { +func NewRouteControllerManager(cfg *config.Config) *OCPRouteControllerManager { s := &OCPRouteControllerManager{} s.configure(cfg) return s @@ -55,13 +55,13 @@ func (s *OCPRouteControllerManager) Dependencies() []string { return []string{"kube-apiserver", "openshift-crd-manager"} } -func (s *OCPRouteControllerManager) configure(cfg *config.MicroshiftConfig) { +func (s *OCPRouteControllerManager) configure(cfg *config.Config) { s.kubeconfig = cfg.KubeConfigPath(config.RouteControllerManager) s.kubeadmconfig = cfg.KubeConfigPath(config.KubeAdmin) s.config = s.writeConfig(cfg) } -func (s *OCPRouteControllerManager) writeConfig(cfg *config.MicroshiftConfig) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { +func (s *OCPRouteControllerManager) writeConfig(cfg *config.Config) *openshiftcontrolplanev1.OpenShiftControllerManagerConfig { servingCertDir := cryptomaterial.RouteControllerManagerServingCertDir(cryptomaterial.CertsDirectory(microshiftDataDir)) c := &openshiftcontrolplanev1.OpenShiftControllerManagerConfig{ diff --git a/pkg/controllers/version.go b/pkg/controllers/version.go index adb7e04c58..4495cbcc8f 100644 --- a/pkg/controllers/version.go +++ b/pkg/controllers/version.go @@ -25,10 +25,10 @@ import ( ) type VersionManager struct { - cfg *config.MicroshiftConfig + cfg *config.Config } -func NewVersionManager(cfg *config.MicroshiftConfig) *VersionManager { +func NewVersionManager(cfg *config.Config) *VersionManager { s := &VersionManager{} s.cfg = cfg return s diff --git a/pkg/kustomize/apply.go b/pkg/kustomize/apply.go index bfe06de1c6..c87453bd86 100644 --- a/pkg/kustomize/apply.go +++ b/pkg/kustomize/apply.go @@ -30,7 +30,7 @@ type Kustomizer struct { kubeconfig string } -func NewKustomizer(cfg *config.MicroshiftConfig) *Kustomizer { +func NewKustomizer(cfg *config.Config) *Kustomizer { return &Kustomizer{ paths: microshiftManifestsDir, kubeconfig: cfg.KubeConfigPath(config.KubeAdmin), diff --git a/pkg/loadbalancerservice/controller.go b/pkg/loadbalancerservice/controller.go index 56fd397051..3b94100ef6 100644 --- a/pkg/loadbalancerservice/controller.go +++ b/pkg/loadbalancerservice/controller.go @@ -34,9 +34,9 @@ type LoadbalancerServiceController struct { var _ servicemanager.Service = &LoadbalancerServiceController{} -func NewLoadbalancerServiceController(cfg *config.MicroshiftConfig) *LoadbalancerServiceController { +func NewLoadbalancerServiceController(cfg *config.Config) *LoadbalancerServiceController { return &LoadbalancerServiceController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), } } diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 7ff7e62b9b..29d7231aaf 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -23,10 +23,10 @@ type MicroShiftmDNSController struct { stopCh chan struct{} } -func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController { +func NewMicroShiftmDNSController(cfg *config.Config) *MicroShiftmDNSController { return &MicroShiftmDNSController{ - NodeIP: cfg.NodeIP, - NodeName: cfg.NodeName, + NodeIP: cfg.Node.NodeIP, + NodeName: cfg.Node.HostnameOverride, KubeConfig: cfg.KubeConfigPath(config.KubeAdmin), hostCount: make(map[string]int), } diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 562c49dad8..55ea2b1695 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -51,7 +51,7 @@ type KubeletServer struct { kubeconfig *kubeletconfig.KubeletConfiguration } -func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { +func NewKubeletServer(cfg *config.Config) *KubeletServer { s := &KubeletServer{} s.configure(cfg) return s @@ -60,7 +60,7 @@ func NewKubeletServer(cfg *config.MicroshiftConfig) *KubeletServer { func (s *KubeletServer) Name() string { return componentKubelet } func (s *KubeletServer) Dependencies() []string { return []string{"kube-apiserver"} } -func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { +func (s *KubeletServer) configure(cfg *config.Config) { if err := s.writeConfig(cfg); err != nil { klog.Fatalf("Failed to write kubelet config", err) @@ -74,8 +74,8 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { kubeletFlags.BootstrapKubeconfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.KubeConfig = cfg.KubeConfigPath(config.Kubelet) kubeletFlags.RuntimeCgroups = "/system.slice/crio.service" - kubeletFlags.HostnameOverride = cfg.NodeName - kubeletFlags.NodeIP = cfg.NodeIP + kubeletFlags.HostnameOverride = cfg.Node.HostnameOverride + kubeletFlags.NodeIP = cfg.Node.NodeIP kubeletFlags.ContainerRuntime = "remote" kubeletFlags.RemoteRuntimeEndpoint = "unix:///var/run/crio/crio.sock" kubeletFlags.NodeLabels["node-role.kubernetes.io/control-plane"] = "" @@ -93,7 +93,7 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) { s.kubeletflags = kubeletFlags } -func (s *KubeletServer) writeConfig(cfg *config.MicroshiftConfig) error { +func (s *KubeletServer) writeConfig(cfg *config.Config) error { certsDir := cryptomaterial.CertsDirectory(microshiftDataDir) servingCertDir := cryptomaterial.KubeletServingCertDir(certsDir) @@ -111,7 +111,7 @@ cgroupDriver: "systemd" failSwapOn: false volumePluginDir: ` + microshiftDataDir + `/kubelet-plugins/volume/exec clusterDNS: - - ` + cfg.Cluster.DNS + ` + - ` + cfg.Network.DNS + ` clusterDomain: cluster.local containerLogMaxSize: 50Mi maxPods: 250 diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index e43813d7f8..a202bc5a04 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -38,7 +38,7 @@ type NetworkConfiguration struct { skipInterfaceConfiguration bool } -func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration { +func NewNetworkConfiguration(cfg *config.Config) *NetworkConfiguration { n := &NetworkConfiguration{} n.configure(cfg) return n @@ -47,9 +47,9 @@ func NewNetworkConfiguration(cfg *config.MicroshiftConfig) *NetworkConfiguration func (n *NetworkConfiguration) Name() string { return componentNetworkConfiguration } func (n *NetworkConfiguration) Dependencies() []string { return []string{} } -func (n *NetworkConfiguration) configure(cfg *config.MicroshiftConfig) { - n.kasAdvertiseAddress = cfg.KASAdvertiseAddress - n.skipInterfaceConfiguration = cfg.SkipKASInterface +func (n *NetworkConfiguration) configure(cfg *config.Config) { + n.kasAdvertiseAddress = cfg.ApiServer.AdvertiseAddress + n.skipInterfaceConfiguration = cfg.ApiServer.SkipInterface } func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, stopped chan<- struct{}) error { diff --git a/pkg/sysconfwatch/sysconfwatch_linux.go b/pkg/sysconfwatch/sysconfwatch_linux.go index b68d159bd6..02f43e7018 100644 --- a/pkg/sysconfwatch/sysconfwatch_linux.go +++ b/pkg/sysconfwatch/sysconfwatch_linux.go @@ -36,7 +36,7 @@ type SysConfWatchController struct { timerFd int } -func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchController { +func NewSysConfWatchController(cfg *config.Config) *SysConfWatchController { // Create a realtime clock timer with asynchronous read support fd, err := unix.TimerfdCreate(unix.CLOCK_REALTIME, unix.TFD_CLOEXEC|unix.TFD_NONBLOCK) if err != nil { @@ -55,7 +55,7 @@ func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchContro } return &SysConfWatchController{ - NodeIP: cfg.NodeIP, + NodeIP: cfg.Node.NodeIP, timerFd: fd, } }