Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions docs/howto_config.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ cluster:
domain: ""
url: ""
mtu: ""
nodeIP: ""
nodeName: ""
logVLevel: ""
```
Expand All @@ -34,7 +33,6 @@ The configuration settings alongside with the supported command line arguments a
| domain | --cluster-domain | MICROSHIFT_CLUSTER_DOMAIN | Base DNS domain used to construct fully qualified pod and service domain names
| url | --url | MICROSHIFT_CLUSTER_URL | URL of the API server for the cluster.
| mtu | --cluster-mtu | MICROSHIFT_CLUSTER_MTU | The maximum transmission unit for the Generic Network Virtualization Encapsulation overlay network
| nodeIP | --node-ip | MICROSHIFT_NODEIP | The IP address of the node, defaults to IP of the default route
| nodeName | --node-name | MICROSHIFT_NODENAME | The name of the node, defaults to hostname
| logVLevel | --v | MICROSHIFT_LOGVLEVEL | Log verbosity (0-5)

Expand All @@ -51,7 +49,6 @@ cluster:
domain: cluster.local
url: https://127.0.0.1:6443
mtu: "1400"
nodeIP: ""
nodeName: ""
logVLevel: 0
```
Expand Down
11 changes: 0 additions & 11 deletions packaging/microshift/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,8 @@ cluster:
# MTU for CNI
#mtu: "1400"

# Location for data created by MicroShift
#dataDir: /var/lib/microshift

# Log verbosity (0-5)
#logVLevel: 0

# Locations to scan for manifests to load on startup
#manifests:
#- /usr/lib/microshift/manifests
#- /etc/microshift/manifests

# The IP of the node (defaults to IP of default route)
#nodeIP: ""

# The name of the node (defaults to hostname)
#nodeName: ""
13 changes: 8 additions & 5 deletions pkg/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ import (
"github.com/openshift/microshift/pkg/util/cryptomaterial"
)

var microshiftDataDir = config.GetDataDir()
var (
microshiftDataDir = config.GetDataDir()
nodeIP = config.GetNodeIP()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of defining that globally every time, could we perhaps create a structure that could be passed on to the various init functions alongside the configuration?

)

func initAll(cfg *config.MicroshiftConfig) error {
// create CA and keys
Expand Down Expand Up @@ -148,7 +151,7 @@ func initCerts(cfg *config.MicroshiftConfig) ([]byte, *cryptomaterial.Certificat
Name: "kubelet-server",
ValidityDays: cryptomaterial.ServingCertValidityDays,
},
Hostnames: []string{cfg.NodeName, cfg.NodeIP},
Hostnames: []string{cfg.NodeName, nodeIP},
},
),
),
Expand Down Expand Up @@ -208,15 +211,15 @@ func initCerts(cfg *config.MicroshiftConfig) ([]byte, *cryptomaterial.Certificat
ValidityDays: 3 * 365,
},
UserInfo: &user.DefaultInfo{Name: "system:etcd-peer:etcd-client", Groups: []string{"system:etcd-peers"}},
Hostnames: []string{"localhost", cfg.NodeIP, "127.0.0.1", cfg.NodeName},
Hostnames: []string{"localhost", nodeIP, "127.0.0.1", cfg.NodeName},
},
&cryptomaterial.PeerCertificateSigningRequestInfo{
CertificateSigningRequestInfo: cryptomaterial.CertificateSigningRequestInfo{
Name: "etcd-serving",
ValidityDays: 3 * 365,
},
UserInfo: &user.DefaultInfo{Name: "system:etcd-server:etcd-client", Groups: []string{"system:etcd-servers"}},
Hostnames: []string{"localhost", "127.0.0.1", cfg.NodeIP, cfg.NodeName},
Hostnames: []string{"localhost", "127.0.0.1", nodeIP, cfg.NodeName},
},
),
).WithCABundle(
Expand Down Expand Up @@ -255,7 +258,7 @@ func initCerts(cfg *config.MicroshiftConfig) ([]byte, *cryptomaterial.Certificat
// kube-apiserver
if err := util.GenCerts("kube-apiserver", filepath.Join(microshiftDataDir, "/certs/kube-apiserver/secrets/service-network-serving-certkey"),
"tls.crt", "tls.key",
[]string{"kube-apiserver", cfg.NodeIP, cfg.NodeName, "127.0.0.1", "kubernetes.default.svc", "kubernetes.default", "kubernetes",
[]string{"kube-apiserver", nodeIP, cfg.NodeName, "127.0.0.1", "kubernetes.default.svc", "kubernetes.default", "kubernetes",
"localhost",
apiServerServiceIP.String()}); err != nil {
return nil, nil, err
Expand Down
4 changes: 1 addition & 3 deletions pkg/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ func addRunFlags(cmd *cobra.Command, cfg *config.MicroshiftConfig) {
// All other flags will be read after reading both config file and env vars.
flags.StringSlice("roles", cfg.Roles, "The roles of this MicroShift instance.")
flags.String("node-name", cfg.NodeName, "The hostname of the node.")
flags.String("node-ip", cfg.NodeIP, "The IP address of the node.")
flags.String("url", cfg.Cluster.URL, "The URL of the API server.")
flags.String("cluster-cidr", cfg.Cluster.ClusterCIDR, "The IP range in CIDR notation for pods in the cluster.")
flags.String("service-cidr", cfg.Cluster.ServiceCIDR, "The IP range in CIDR notation for services in the cluster.")
Expand Down Expand Up @@ -77,7 +76,6 @@ func RunMicroshift(cfg *config.MicroshiftConfig, flags *pflag.FlagSet) error {
// see https://github.com/openshift/microshift/pull/471

if err := util.AddToNoProxyEnv(
cfg.NodeIP,
cfg.NodeName,
cfg.Cluster.ClusterCIDR,
cfg.Cluster.ServiceCIDR,
Expand Down Expand Up @@ -155,7 +153,7 @@ func RunMicroshift(cfg *config.MicroshiftConfig, flags *pflag.FlagSet) error {
go func() {
var server *http.Server
server = &http.Server{
Addr: cfg.NodeIP + ":29500",
Addr: nodeIP + ":29500",
Handler: mux,
}
err := server.ListenAndServe()
Expand Down
1 change: 0 additions & 1 deletion pkg/cmd/showConfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ func NewShowConfigCommand(ioStreams genericclioptions.IOStreams) *cobra.Command

switch opts.Mode {
case "default":
cfg.NodeIP = ""
cfg.NodeName = ""
case "effective":
// Load the current configuration
Expand Down
5 changes: 4 additions & 1 deletion pkg/components/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ import (
"k8s.io/klog/v2"
)

var microshiftDataDir = config.GetDataDir()
var (
microshiftDataDir = config.GetDataDir()
nodeIP = config.GetNodeIP()
)

func StartComponents(cfg *config.MicroshiftConfig) error {
kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin)
Expand Down
2 changes: 1 addition & 1 deletion pkg/components/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func renderParamsFromConfig(cfg *config.MicroshiftConfig, extra assets.RenderPar
params := map[string]interface{}{
"ReleaseImage": release.Image,
"NodeName": cfg.NodeName,
"NodeIP": cfg.NodeIP,
"NodeIP": nodeIP,
"ClusterCIDR": cfg.Cluster.ClusterCIDR,
"ServiceCIDR": cfg.Cluster.ServiceCIDR,
"ClusterDNS": cfg.Cluster.DNS,
Expand Down
22 changes: 13 additions & 9 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ var (
configFile = findConfigFile()
dataDir = findDataDir()
manifestsDir = findManifestsDir()
nodeIP = findNodeIP()
)

type ClusterConfig struct {
Expand All @@ -60,7 +61,6 @@ type MicroshiftConfig struct {
Roles []string `json:"roles"`

NodeName string `json:"nodeName"`
NodeIP string `json:"nodeIP"`

Cluster ClusterConfig `json:"cluster"`
Debug DebugConfig `json:"debug"`
Expand All @@ -78,6 +78,18 @@ func GetManifestsDir() []string {
return manifestsDir
}

func GetNodeIP() string {
return nodeIP
}

func findNodeIP() string {
hostIP, err := util.GetHostIP()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We expect hosts to have multiple IPs. How does the user influence how this function picks the IP to use?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems the nodeIP option is not repected in main branch, even it is user configurable. We probably want to fix it. This influence approach shall also work for microshift-ovs-init service script, which sets up the nodeIP interface for ovn-kubernetes CNI plugin.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In which scenario would a user want to pick a specific IP? The IP to which core components of microshift bind to shouldn't really matter to the end-user since their applications will most likely only communicate via the different services.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I remember that some popular distros also supported specifying the interface name or something like IP(eth0) to make it work with DHCP too

Copy link
Member

@dgrisonnet dgrisonnet Oct 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I am not mistaken, the nodeIP is not even configurable in OCP today. The closest thing I could find is this effort: https://github.com/openshift/enhancements/blob/ef85659d01738b9f89958d5f0da31cff05bb1182/enhancements/network/ip-interface-selection.md but this seem way too complex for MicroShift to need something like that right now.

I remember that some popular distros also supported specifying the interface name or something like IP(eth0) to make it work with DHCP too

I am not sure why that would be beneficial to do for control plane components.

Another concern I have is what would happen if microshift was first started with a nodeIP, but then a user decided to change it on the fly. Everything that used to depend on the previous nodeIP will not be able to reach the new nodeIP if we don't have a way to propagate the update. I don't think that's supported in k8s in general since one would most likely have to delete the node to update the IP on the various components.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unlike OCP hosts, we have to deal with the fact that a host running MicroShift may change its IP or move between networks. So we either need a way to pick a node IP that will never change, or we need to support it being updated. Could we always use a localhost IP or the hardware's IPv6 local address, for example? What effect would that have? What needs to use the node IP?

if err != nil {
klog.Fatalf("failed to get host IP: %v", err)
}
return hostIP
}

// KubeConfigID identifies the different kubeconfigs managed in the DataDir
type KubeConfigID string

Expand All @@ -98,18 +110,13 @@ func NewMicroshiftConfig() *MicroshiftConfig {
if err != nil {
klog.Fatalf("Failed to get hostname %v", err)
}
nodeIP, err := util.GetHostIP()
if err != nil {
klog.Fatalf("failed to get host IP: %v", err)
}

defaultRoles := make([]string, len(validRoles))
copy(defaultRoles, validRoles)
return &MicroshiftConfig{
LogVLevel: 0,
Roles: defaultRoles,
NodeName: nodeName,
NodeIP: nodeIP,
Cluster: ClusterConfig{
URL: "https://127.0.0.1:6443",
ClusterCIDR: "10.42.0.0/16",
Expand Down Expand Up @@ -220,9 +227,6 @@ func (c *MicroshiftConfig) ReadFromCmdLine(flags *pflag.FlagSet) error {
if s, err := flags.GetString("node-name"); err == nil && flags.Changed("node-name") {
c.NodeName = s
}
if s, err := flags.GetString("node-ip"); err == nil && flags.Changed("node-ip") {
c.NodeIP = s
}
if s, err := flags.GetString("url"); err == nil && flags.Changed("url") {
c.Cluster.URL = s
}
Expand Down
7 changes: 0 additions & 7 deletions pkg/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ func TestCommandLineConfig(t *testing.T) {
LogVLevel: 4,
Roles: []string{"controlplane", "node"},
NodeName: "node1",
NodeIP: "1.2.3.4",
Cluster: ClusterConfig{
URL: "https://1.2.3.4:6443",
ClusterCIDR: "10.20.30.40/16",
Expand All @@ -71,7 +70,6 @@ func TestCommandLineConfig(t *testing.T) {
flags.Int("v", config.LogVLevel, "")
flags.StringSlice("roles", config.Roles, "")
flags.String("node-name", config.NodeName, "")
flags.String("node-ip", config.NodeIP, "")
flags.String("url", config.Cluster.URL, "")
flags.String("cluster-cidr", config.Cluster.ClusterCIDR, "")
flags.String("service-cidr", config.Cluster.ServiceCIDR, "")
Expand All @@ -87,7 +85,6 @@ func TestCommandLineConfig(t *testing.T) {
"--v=" + strconv.Itoa(tt.config.LogVLevel),
"--roles=" + strings.Join(tt.config.Roles, ","),
"--node-name=" + tt.config.NodeName,
"--node-ip=" + tt.config.NodeIP,
"--url=" + tt.config.Cluster.URL,
"--cluster-cidr=" + tt.config.Cluster.ClusterCIDR,
"--service-cidr=" + tt.config.Cluster.ServiceCIDR,
Expand Down Expand Up @@ -128,7 +125,6 @@ func TestEnvironmentVariableConfig(t *testing.T) {
LogVLevel: 23,
Roles: []string{"controlplane", "node"},
NodeName: "node1",
NodeIP: "1.2.3.4",
Cluster: ClusterConfig{
URL: "https://cluster.com:4343/endpoint",
ClusterCIDR: "10.20.30.40/16",
Expand All @@ -147,7 +143,6 @@ func TestEnvironmentVariableConfig(t *testing.T) {
{"MICROSHIFT_LOGVLEVEL", "23"},
{"MICROSHIFT_ROLES", "controlplane,node"},
{"MICROSHIFT_NODENAME", "node1"},
{"MICROSHIFT_NODEIP", "1.2.3.4"},
{"MICROSHIFT_CLUSTER_URL", "https://cluster.com:4343/endpoint"},
{"MICROSHIFT_CLUSTER_CLUSTERCIDR", "10.20.30.40/16"},
{"MICROSHIFT_CLUSTER_SERVICECIDR", "40.30.20.10/16"},
Expand All @@ -162,7 +157,6 @@ func TestEnvironmentVariableConfig(t *testing.T) {
LogVLevel: 23,
Roles: []string{"controlplane", "node"},
NodeName: "node1",
NodeIP: "1.2.3.4",
Cluster: ClusterConfig{
URL: "https://cluster.com:4343/endpoint",
ClusterCIDR: "10.20.30.40/16",
Expand All @@ -181,7 +175,6 @@ func TestEnvironmentVariableConfig(t *testing.T) {
{"MICROSHIFT_LOGVLEVEL", "23"},
{"MICROSHIFT_ROLES", "controlplane,node"},
{"MICROSHIFT_NODENAME", "node1"},
{"MICROSHIFT_NODEIP", "1.2.3.4"},
{"MICROSHIFT_CLUSTER_URL", "https://cluster.com:4343/endpoint"},
{"MICROSHIFT_CLUSTER_CLUSTERCIDR", "10.20.30.40/16"},
{"MICROSHIFT_CLUSTER_SERVICECIDR", "40.30.20.10/16"},
Expand Down
11 changes: 6 additions & 5 deletions pkg/controllers/etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ var (
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
}
microshiftDataDir = config.GetDataDir()
nodeIP = config.GetNodeIP()
)

const (
Expand Down Expand Up @@ -70,14 +71,14 @@ func (s *EtcdService) configure(cfg *config.MicroshiftConfig) {
//s.etcdCfg.ForceNewCluster = true //TODO
s.etcdCfg.Logger = "zap"
s.etcdCfg.Dir = dataDir
s.etcdCfg.APUrls = setURL([]string{cfg.NodeIP}, ":2380")
s.etcdCfg.LPUrls = setURL([]string{cfg.NodeIP}, ":2380")
s.etcdCfg.ACUrls = setURL([]string{cfg.NodeIP}, ":2379")
s.etcdCfg.LCUrls = setURL([]string{"127.0.0.1", cfg.NodeIP}, ":2379")
s.etcdCfg.APUrls = setURL([]string{nodeIP}, ":2380")
s.etcdCfg.LPUrls = setURL([]string{nodeIP}, ":2380")
s.etcdCfg.ACUrls = setURL([]string{nodeIP}, ":2379")
s.etcdCfg.LCUrls = setURL([]string{"127.0.0.1", nodeIP}, ":2379")
s.etcdCfg.ListenMetricsUrls = setURL([]string{"127.0.0.1"}, ":2381")

s.etcdCfg.Name = cfg.NodeName
s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.NodeName, cfg.NodeIP)
s.etcdCfg.InitialCluster = fmt.Sprintf("%s=https://%s:2380", cfg.NodeName, nodeIP)

s.etcdCfg.CipherSuites = tlsCipherSuites
s.etcdCfg.ClientTLSInfo.CertFile = cryptomaterial.PeerCertPath(etcdServingCertDir)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/kube-apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func (s *KubeAPIServer) configure(cfg *config.MicroshiftConfig) error {

overrides := &kubecontrolplanev1.KubeAPIServerConfig{
APIServerArguments: map[string]kubecontrolplanev1.Arguments{
"advertise-address": {cfg.NodeIP},
"advertise-address": {nodeIP},
"audit-policy-file": {microshiftDataDir + "/resources/kube-apiserver-audit-policies/default.yaml"},
"client-ca-file": {clientCABundlePath},
"etcd-cafile": {cryptomaterial.CACertPath(cryptomaterial.EtcdSignerDir(certsDir))},
Expand Down
2 changes: 1 addition & 1 deletion pkg/mdns/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ type MicroShiftmDNSController struct {

func NewMicroShiftmDNSController(cfg *config.MicroshiftConfig) *MicroShiftmDNSController {
return &MicroShiftmDNSController{
NodeIP: cfg.NodeIP,
NodeIP: config.GetNodeIP(),
NodeName: cfg.NodeName,
KubeConfig: cfg.KubeConfigPath(config.KubeAdmin),
hostCount: make(map[string]int),
Expand Down
2 changes: 1 addition & 1 deletion pkg/node/kubelet.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (s *KubeletServer) configure(cfg *config.MicroshiftConfig) {
kubeletFlags.BootstrapKubeconfig = cfg.KubeConfigPath(config.Kubelet)
kubeletFlags.KubeConfig = cfg.KubeConfigPath(config.Kubelet)
kubeletFlags.RuntimeCgroups = "/system.slice/crio.service"
kubeletFlags.NodeIP = cfg.NodeIP
kubeletFlags.NodeIP = config.GetNodeIP()
kubeletFlags.ContainerRuntime = "remote"
kubeletFlags.RemoteRuntimeEndpoint = "unix:///var/run/crio/crio.sock"
for _, role := range cfg.Roles {
Expand Down
2 changes: 1 addition & 1 deletion pkg/sysconfwatch/sysconfwatch_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func NewSysConfWatchController(cfg *config.MicroshiftConfig) *SysConfWatchContro
}

return &SysConfWatchController{
NodeIP: cfg.NodeIP,
NodeIP: config.GetNodeIP(),
timerFd: fd,
}
}
Expand Down
1 change: 0 additions & 1 deletion test/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ roles:
- role1
- role2
nodeName: node1
nodeIP: '1.2.3.4'
cluster:
url: https://1.2.3.4:6443
clusterCIDR: '10.20.30.40/16'
Expand Down