OSASINFRA-3075 - openstack: external LB support

data/data/openstack/bootstrap/main.tf

@@ -52,8 +52,11 @@ resource "openstack_networking_port_v2" "bootstrap_port" {
     subnet_id = var.nodes_subnet_id
   }
 
-  allowed_address_pairs {
-    ip_address = var.openstack_api_int_ip
+  dynamic "allowed_address_pairs" {
+    for_each = var.openstack_api_int_ip == "" ? []: [1]
+    content {
+      ip_address = var.openstack_api_int_ip
+    }
   }
 
   depends_on = [var.master_port_ids]

data/data/openstack/masters/private-network.tf

@@ -62,18 +62,25 @@ resource "openstack_networking_port_v2" "masters" {
     subnet_id = local.nodes_subnet_id
   }
 
-  allowed_address_pairs {
-    ip_address = var.openstack_api_int_ip
+  dynamic "allowed_address_pairs" {
+    for_each = var.openstack_api_int_ip == "" ? []: [1]
+    content {
+      ip_address = var.openstack_api_int_ip
+    }
   }
 
-  allowed_address_pairs {
-    ip_address = var.openstack_ingress_ip
+  dynamic "allowed_address_pairs" {
+    for_each = var.openstack_ingress_ip == "" ? []: [1]
+    content {
+      ip_address = var.openstack_ingress_ip
+    }
   }
 
   depends_on = [openstack_networking_port_v2.api_port, openstack_networking_port_v2.ingress_port]
 }
 
 resource "openstack_networking_port_v2" "api_port" {
+  count       = var.openstack_api_int_ip == "" ? 0 : 1
   name        = "${var.cluster_id}-api-port"
   description = local.description
 
@@ -89,6 +96,7 @@ resource "openstack_networking_port_v2" "api_port" {
 }
 
 resource "openstack_networking_port_v2" "ingress_port" {
+  count       = var.openstack_ingress_ip == "" ? 0 : 1
   name        = "${var.cluster_id}-ingress-port"
   description = local.description
 
@@ -134,15 +142,15 @@ resource "openstack_networking_trunk_v2" "masters" {
 // as expected.
 
 resource "openstack_networking_floatingip_associate_v2" "api_fip" {
-  count       = length(var.openstack_api_floating_ip) == 0 ? 0 : 1
-  port_id     = openstack_networking_port_v2.api_port.id
+  count       = (var.openstack_api_int_ip == "" || length(var.openstack_api_floating_ip) == 0) ? 0 : 1
+  port_id     = openstack_networking_port_v2.api_port[0].id
   floating_ip = var.openstack_api_floating_ip
   depends_on  = [openstack_networking_router_interface_v2.nodes_router_interface]
 }
 
 resource "openstack_networking_floatingip_associate_v2" "ingress_fip" {
-  count       = length(var.openstack_ingress_floating_ip) == 0 ? 0 : 1
-  port_id     = openstack_networking_port_v2.ingress_port.id
+  count       = (var.openstack_ingress_ip == "" || length(var.openstack_ingress_floating_ip) == 0) ? 0 : 1
+  port_id     = openstack_networking_port_v2.ingress_port[0].id
   floating_ip = var.openstack_ingress_floating_ip
   depends_on  = [openstack_networking_router_interface_v2.nodes_router_interface]
 }

data/data/openstack/variables-openstack.tf

@@ -287,11 +287,13 @@ EOF
 variable "openstack_api_int_ip" {
   type        = string
   description = "IP on the node subnet reserved for api-int VIP."
+  default     = ""
 }
 
 variable "openstack_ingress_ip" {
   type        = string
   description = "IP on the nodes subnet reserved for the ingress VIP."
+  default     = ""
 }
 
 variable "openstack_external_dns" {

pkg/asset/ignition/machine/node.go

@@ -39,7 +39,9 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
 			ignitionHost = net.JoinHostPort(installConfig.Nutanix.APIVIPs[0], "22623")
 		}
 	case openstacktypes.Name:
-		ignitionHost = net.JoinHostPort(installConfig.OpenStack.APIVIPs[0], "22623")
+		if len(installConfig.OpenStack.APIVIPs) > 0 {
+			ignitionHost = net.JoinHostPort(installConfig.OpenStack.APIVIPs[0], "22623")
+		}
 	case ovirttypes.Name:
 		ignitionHost = net.JoinHostPort(installConfig.Ovirt.APIVIPs[0], "22623")
 	case vspheretypes.Name:

pkg/asset/installconfig/openstack/validate.go

@@ -7,6 +7,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
 	"github.com/openshift/installer/pkg/asset/installconfig/openstack/validation"
+	vspherevalidation "github.com/openshift/installer/pkg/asset/installconfig/vsphere"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 	openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults"
@@ -59,6 +60,20 @@ func Validate(ic *types.InstallConfig) error {
 		allErrs = append(allErrs, validation.ValidateMachinePool(&compute, ci, false, fldPath.Child("platform", "openstack"))...)
 	}
 
+	// If APIVIPs and IngressVIPs is equal to zero
+	// then don't validate the VIPs.
+	// Instead, ensure there is a configured
+	// DNS record for api and test if the load
+	// balancer is configured.
+	// The VIP parameters within the Infrastructure status object
+	// will be empty. This will cause MCO to not deploy
+	// the static pods: haproxy, keepalived and coredns.
+	// This will allow the use of an external load balancer
+	// and RHCOS nodes to be on multiple L2 segments.
+	if len(ic.Platform.OpenStack.APIVIPs) == 0 && len(ic.Platform.OpenStack.IngressVIPs) == 0 {
+		allErrs = append(allErrs, vspherevalidation.EnsureLoadBalancerDNS(ic, field.NewPath("platform"))...)
+	}
+
 	return allErrs.ToAggregate()
 }
 

pkg/asset/installconfig/vsphere/validation.go

@@ -84,7 +84,7 @@ func ValidateMultiZoneForProvisioning(ic *types.InstallConfig) error {
 	// This will allow the use of an external load balancer
 	// and RHCOS nodes to be on multiple L2 segments.
 	if len(ic.Platform.VSphere.APIVIPs) == 0 && len(ic.Platform.VSphere.IngressVIPs) == 0 {
-		allErrs = append(allErrs, ensureLoadBalancerDNS(ic, field.NewPath("platform"))...)
+		allErrs = append(allErrs, EnsureLoadBalancerDNS(ic, field.NewPath("platform"))...)
 	}
 
 	var clients = make(map[string]*validationContext, 0)
@@ -427,7 +427,7 @@ func validateVcenterPrivileges(validationCtx *validationContext, fldPath *field.
 	return field.ErrorList{}
 }
 
-func ensureLoadBalancerDNS(installConfig *types.InstallConfig, fldPath *field.Path) field.ErrorList {
+func EnsureLoadBalancerDNS(installConfig *types.InstallConfig, fldPath *field.Path) field.ErrorList {
 	var lastErr error
 	var uris []string
 	dialTimeout := time.Second

pkg/asset/manifests/infrastructure.go

@@ -197,11 +197,13 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 		config.Spec.PlatformSpec.Type = configv1.NonePlatformType
 	case openstack.Name:
 		config.Spec.PlatformSpec.Type = configv1.OpenStackPlatformType
-		config.Status.PlatformStatus.OpenStack = &configv1.OpenStackPlatformStatus{
-			APIServerInternalIP:  installConfig.Config.OpenStack.APIVIPs[0],
-			IngressIP:            installConfig.Config.OpenStack.IngressVIPs[0],
-			APIServerInternalIPs: installConfig.Config.OpenStack.APIVIPs,
-			IngressIPs:           installConfig.Config.OpenStack.IngressVIPs,
+		if len(installConfig.Config.OpenStack.APIVIPs) > 0 {
+			config.Status.PlatformStatus.OpenStack = &configv1.OpenStackPlatformStatus{
+				APIServerInternalIP:  installConfig.Config.OpenStack.APIVIPs[0],
+				IngressIP:            installConfig.Config.OpenStack.IngressVIPs[0],
+				APIServerInternalIPs: installConfig.Config.OpenStack.APIVIPs,
+				IngressIPs:           installConfig.Config.OpenStack.IngressVIPs,
+			}
 		}
 	case vsphere.Name:
 		config.Spec.PlatformSpec.Type = configv1.VSpherePlatformType

pkg/tfvars/openstack/openstack.go

@@ -186,6 +186,12 @@ func TFVars(
 		}
 	}
 
+	var apiVIP, ingressVIP string
+	if len(installConfig.Config.Platform.OpenStack.APIVIPs) > 0 {
+		apiVIP = installConfig.Config.Platform.OpenStack.APIVIPs[0]
+		ingressVIP = installConfig.Config.Platform.OpenStack.IngressVIPs[0]
+	}
+
 	return json.MarshalIndent(struct {
 		BaseImageName                     string                            `json:"openstack_base_image_name,omitempty"`
 		ExternalNetwork                   string                            `json:"openstack_external_network,omitempty"`
@@ -218,8 +224,8 @@ func TFVars(
 		FlavorName:                        masterSpecs[0].Flavor,
 		APIFloatingIP:                     installConfig.Config.Platform.OpenStack.APIFloatingIP,
 		IngressFloatingIP:                 installConfig.Config.Platform.OpenStack.IngressFloatingIP,
-		APIVIP:                            installConfig.Config.Platform.OpenStack.APIVIPs[0],
-		IngressVIP:                        installConfig.Config.Platform.OpenStack.IngressVIPs[0],
+		APIVIP:                            apiVIP,
+		IngressVIP:                        ingressVIP,
 		TrunkSupport:                      masterSpecs[0].Trunk,
 		OctaviaSupport:                    octaviaSupport,
 		RootVolumeSize:                    rootVolumeSize,

pkg/types/openstack/defaults/platform.go

@@ -1,11 +1,8 @@
 package defaults
 
 import (
-	"fmt"
 	"os"
 
-	"github.com/apparentlymart/go-cidr/cidr"
-
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 )
@@ -23,32 +20,4 @@ func SetPlatformDefaults(p *openstack.Platform, n *types.Networking) {
 			p.Cloud = DefaultCloudName
 		}
 	}
-	// APIVIP returns the internal virtual IP address (VIP) put in front
-	// of the Kubernetes API server for use by components inside the
-	// cluster. The DNS static pods running on the nodes resolve the
-	// api-int record to APIVIP.
-	if len(p.APIVIPs) == 0 && p.DeprecatedAPIVIP == "" {
-		vip, err := cidr.Host(&n.MachineNetwork[0].CIDR.IPNet, 5)
-		if err != nil {
-			// This will fail validation and abort the install
-			p.APIVIPs = []string{fmt.Sprintf("could not derive API VIP from machine networks: %s", err.Error())}
-		} else {
-			p.APIVIPs = []string{vip.String()}
-		}
-	}
-
-	// IngressVIP returns the internal virtual IP address (VIP) put in
-	// front of the OpenShift router pods. This provides the internal
-	// accessibility to the internal pods running on the worker nodes,
-	// e.g. `console`. The DNS static pods running on the nodes resolve
-	// the wildcard apps record to IngressVIP.
-	if len(p.IngressVIPs) == 0 && p.DeprecatedIngressVIP == "" {
-		vip, err := cidr.Host(&n.MachineNetwork[0].CIDR.IPNet, 7)
-		if err != nil {
-			// This will fail validation and abort the install
-			p.IngressVIPs = []string{fmt.Sprintf("could not derive Ingress VIP from machine networks: %s", err.Error())}
-		} else {
-			p.IngressVIPs = []string{vip.String()}
-		}
-	}
 }

pkg/types/validation/installconfig.go

@@ -503,7 +503,7 @@ func validateVIPsForPlatform(network *types.Networking, platform *types.Platform
 			Ingress: platform.OpenStack.IngressVIPs,
 		}
 
-		allErrs = append(allErrs, validateAPIAndIngressVIPs(virtualIPs, newVIPsFields, true, network, fldPath.Child(openstack.Name))...)
+		allErrs = append(allErrs, validateAPIAndIngressVIPs(virtualIPs, newVIPsFields, false, network, fldPath.Child(openstack.Name))...)
 	case platform.VSphere != nil:
 		allErrs = append(allErrs, ensureIPv4IsFirstInDualStackSlice(&platform.VSphere.APIVIPs, fldPath.Child(vsphere.Name, newVIPsFields.APIVIPs))...)
 		allErrs = append(allErrs, ensureIPv4IsFirstInDualStackSlice(&platform.VSphere.IngressVIPs, fldPath.Child(vsphere.Name, newVIPsFields.IngressVIPs))...)