USHIFT-3478: Configuration adaptations for dual stack

pkg/components/render.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"text/template"
 
 	"sigs.k8s.io/yaml"
@@ -25,8 +26,8 @@ func renderParamsFromConfig(cfg *config.Config, extra assets.RenderParams) asset
 		"ReleaseImage": release.Image,
 		"NodeName":     cfg.CanonicalNodeName(),
 		"NodeIP":       cfg.Node.NodeIP,
-		"ClusterCIDR":  cfg.Network.ClusterNetwork[0],
-		"ServiceCIDR":  cfg.Network.ServiceNetwork[0],
+		"ClusterCIDR":  strings.Join(cfg.Network.ClusterNetwork, ","),
+		"ServiceCIDR":  strings.Join(cfg.Network.ServiceNetwork, ","),
 		"ClusterDNS":   cfg.Network.DNS,
 		"BaseDomain":   cfg.DNS.BaseDomain,
 	}

pkg/config/apiserver.go

@@ -20,6 +20,18 @@ type ApiServer struct {
 	// The URL and Port of the API server cannot be changed by the user.
 	URL  string `json:"-"`
 	Port int    `json:"-"`
+
+	// In dual stack mode, ovnk requires ovn.OVNGatewayInterface to have one IP
+	// per family or else it wont start. When configuring advertiseAddress,
+	// whether that is manual or automatic, this IP is configured in that
+	// bridge afterwards in node package. Since there is only one IP, ovnk will
+	// return an error complaining about the other IP family for the secondary
+	// cluster/service network gateway. This variable holds all the different
+	// IP addresses that ovn.OVNGatewayInterface needs. Note that this IP is
+	// not configurable by users and it will not be used for apiserver
+	// advertising because of dual stack limitations there. This is only to
+	// make ovnk work properly.
+	AdvertiseAddresses []string `json:"-"`
 }
 
 // NamedCertificateEntry provides certificate details

pkg/config/config.go

@@ -294,25 +294,30 @@ func (c *Config) updateComputedValues() error {
 
 	// If we have no advertise address, pick one.
 	if len(c.ApiServer.AdvertiseAddress) == 0 {
-		// unchecked error because this was done when getting cluster DNS
-		_, svcNet, _ := net.ParseCIDR(c.Network.ServiceNetwork[0])
 		// Since the KAS advertise address was not provided we will default to the
 		// next immediate subnet after the service CIDR. This is due to the fact
 		// that using the actual apiserver service IP as an endpoint slice breaks
 		// host network pods trying to reach apiserver, as the VIP 10.43.0.1:443 is
 		// not translated to 10.43.0.1:6443. It remains unchanged and therefore
 		// connects to the ingress router instead, triggering all sorts of errors.
-		prefix := 32
-		if svcNet.IP.To4() == nil {
-			prefix = 128
+		ip, err := firstIPFromNextSubnet(c.Network.ServiceNetwork[0])
+		if err != nil {
+			return fmt.Errorf("unable to compute AdvertiseAddress: %s", err)
 		}
-		nextSubnet, exceed := cidr.NextSubnet(svcNet, prefix)
-		if exceed {
-			return fmt.Errorf("unable to compute next subnet from service CIDR")
+		c.ApiServer.AdvertiseAddress = ip
+	}
+
+	// Use this variable instead, as we may be in dual stack ip an need to
+	// configure one extra IP address in the ovn gateway interface. Pick
+	// the IP family that was not used for the advertise address and add
+	// the first valid IP for the next subnet.
+	c.ApiServer.AdvertiseAddresses = []string{c.ApiServer.AdvertiseAddress}
+	if c.IsIPv4() && c.IsIPv6() {
+		ip, err := firstIPFromNextSubnet(c.Network.ServiceNetwork[1])
+		if err != nil {
+			return fmt.Errorf("unable to compute secondary address for br-ex: %s", err)
 		}
-		// First and last are the same because of the /32 netmask.
-		firstValidIP, _ := cidr.AddressRange(nextSubnet)
-		c.ApiServer.AdvertiseAddress = firstValidIP.String()
+		c.ApiServer.AdvertiseAddresses = append(c.ApiServer.AdvertiseAddresses, ip)
 	}
 
 	c.computeLoggingSetting()
@@ -379,9 +384,14 @@ func (c *Config) validate() error {
 			"openshift.default",
 			"openshift.default.svc",
 			"openshift.default.svc.cluster.local",
-			c.ApiServer.AdvertiseAddress,
 		) {
-			return fmt.Errorf("subjectAltNames must not contain apiserver kubernetes service names or IPs")
+			return fmt.Errorf("subjectAltNames must not contain kubernetes service names")
+		}
+		if stringSliceContains(
+			c.ApiServer.SubjectAltNames,
+			c.ApiServer.AdvertiseAddresses...,
+		) {
+			return fmt.Errorf("subjectAltNames must not contain apiserver advertise address IPs")
 		}
 	}
 
@@ -391,6 +401,13 @@ func (c *Config) validate() error {
 		)
 	}
 
+	if c.ApiServer.SkipInterface {
+		err := checkAdvertiseAddressConfigured(c.ApiServer.AdvertiseAddresses[0])
+		if err != nil {
+			return err
+		}
+	}
+
 	switch c.Ingress.Status {
 	case StatusManaged, StatusRemoved:
 	default:
@@ -411,7 +428,7 @@ func (c *Config) validate() error {
 	}
 
 	if len(c.Ingress.ListenAddress) != 0 {
-		if err := validateRouterListenAddress(c.Ingress.ListenAddress, c.ApiServer.AdvertiseAddress, c.ApiServer.SkipInterface, c.IsIPv4(), c.IsIPv6()); err != nil {
+		if err := validateRouterListenAddress(c.Ingress.ListenAddress, c.ApiServer.AdvertiseAddresses, c.ApiServer.SkipInterface, c.IsIPv4(), c.IsIPv6()); err != nil {
 			return fmt.Errorf("error validating ingress.listenAddress: %w", err)
 		}
 	}
@@ -495,7 +512,7 @@ func checkAdvertiseAddressConfigured(advertiseAddress string) error {
 	return fmt.Errorf("Advertise address: %s not present in any interface", advertiseAddress)
 }
 
-func validateRouterListenAddress(ingressListenAddresses []string, advertiseAddress string, skipInterface, ipv4, ipv6 bool) error {
+func validateRouterListenAddress(ingressListenAddresses []string, advertiseAddresses []string, skipInterface bool, ipv4, ipv6 bool) error {
 	addresses, err := AllowedListeningIPAddresses(ipv4, ipv6)
 	if err != nil {
 		return err
@@ -505,7 +522,7 @@ func validateRouterListenAddress(ingressListenAddresses []string, advertiseAddre
 		return err
 	}
 	for _, entry := range ingressListenAddresses {
-		if entry == advertiseAddress && !skipInterface {
+		if slices.Contains(advertiseAddresses, entry) && !skipInterface {
 			continue
 		}
 		ip := net.ParseIP(entry)
@@ -666,3 +683,20 @@ func validateNetworkStack(cfg *Config) error {
 	}
 	return nil
 }
+
+func firstIPFromNextSubnet(subnet string) (string, error) {
+	_, svcNet, err := net.ParseCIDR(subnet)
+	if err != nil {
+		return "", err
+	}
+	prefix := 32
+	if svcNet.IP.To4() == nil {
+		prefix = 128
+	}
+	nextSubnet, exceed := cidr.NextSubnet(svcNet, prefix)
+	if exceed {
+		return "", fmt.Errorf("unable to compute next subnet from service CIDR")
+	}
+	firstValidIP, _ := cidr.AddressRange(nextSubnet)
+	return firstValidIP.String(), nil
+}

pkg/controllers/kube-apiserver.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -113,7 +114,7 @@ func (s *KubeAPIServer) configure(cfg *config.Config) error {
 
 	s.masterURL = cfg.ApiServer.URL
 	s.servingCAPath = cryptomaterial.ServiceAccountTokenCABundlePath(certsDir)
-	s.advertiseAddress = cfg.ApiServer.AdvertiseAddress
+	s.advertiseAddress = cfg.ApiServer.AdvertiseAddresses[0]
 
 	namedCerts := []configv1.NamedCertificate{
 		{
@@ -138,7 +139,7 @@ func (s *KubeAPIServer) configure(cfg *config.Config) error {
 	if len(cfg.ApiServer.NamedCertificates) > 0 {
 		for _, namedCertsCfg := range cfg.ApiServer.NamedCertificates {
 			//Validate the cert is non-destructive
-			certAllowed, err := util.IsCertAllowed(cfg.ApiServer.AdvertiseAddress, cfg.Network.ClusterNetwork, cfg.Network.ServiceNetwork, namedCertsCfg.CertPath, namedCertsCfg.Names)
+			certAllowed, err := util.IsCertAllowed(cfg.ApiServer.AdvertiseAddresses[0], cfg.Network.ClusterNetwork, cfg.Network.ServiceNetwork, namedCertsCfg.CertPath, namedCertsCfg.Names)
 			if err != nil {
 				klog.Warningf("Failed to read NamedCertificate from %s - ignoring: %v", namedCertsCfg.CertPath, err)
 				continue
@@ -188,6 +189,7 @@ func (s *KubeAPIServer) configure(cfg *config.Config) error {
 			// limitations. For this, we prefer using names and IPs as a fallback, supporting both single
 			// and multi node.
 			"kubelet-preferred-address-types": {"Hostname", "InternalIP"},
+			"service-cluster-ip-range":        {strings.Join(cfg.Network.ServiceNetwork, ",")},
 
 			"proxy-client-cert-file":           {cryptomaterial.ClientCertPath(aggregatorClientCertDir)},
 			"proxy-client-key-file":            {cryptomaterial.ClientKeyPath(aggregatorClientCertDir)},
@@ -250,7 +252,6 @@ func (s *KubeAPIServer) configure(cfg *config.Config) error {
 		ServiceAccountPublicKeyFiles: []string{
 			filepath.Join(config.DataDir, "/resources/kube-apiserver/secrets/service-account-key/service-account.pub"),
 		},
-		ServicesSubnet:        cfg.Network.ServiceNetwork[0],
 		ServicesNodePortRange: cfg.Network.ServiceNodePortRange,
 	}
 

pkg/controllers/kube-controller-manager.go

@@ -88,7 +88,8 @@ func configure(ctx context.Context, cfg *config.Config) (args []string, applyFn
 			"authorization-kubeconfig":         {kubeConfig},
 			"service-account-private-key-file": {kcmServiceAccountPrivateKeyFile()},
 			"allocate-node-cidrs":              {"true"},
-			"cluster-cidr":                     {cfg.Network.ClusterNetwork[0]},
+			"cluster-cidr":                     {strings.Join(cfg.Network.ClusterNetwork, ",")},
+			"service-cluster-ip-range":         {strings.Join(cfg.Network.ServiceNetwork, ",")},
 			"root-ca-file":                     {kcmRootCAFile()},
 			"secure-port":                      {"10257"},
 			"leader-elect":                     {"false"},

pkg/controllers/kube-controller-manager_test.go

@@ -69,6 +69,7 @@ func TestConfigure(t *testing.T) {
 		fmt.Sprintf("--root-ca-file=%s", kcmRootCAFile()),
 		"--secure-port=10257",
 		fmt.Sprintf("--service-account-private-key-file=%s", kcmServiceAccountPrivateKeyFile()),
+		fmt.Sprintf("--service-cluster-ip-range=%s", cfg.Network.ServiceNetwork[0]),
 		fmt.Sprintf("--tls-cipher-suites=%s", strings.Join(crypto.OpenSSLToIANACipherSuites(fixedTLSProfile.Ciphers), ",")),
 		fmt.Sprintf("--tls-min-version=%s", string(fixedTLSProfile.MinTLSVersion)),
 		"--use-service-account-credentials=true",