OSASINFRA-3090: External LB support for on-prem platforms

data/data/install.openshift.io_installconfigs.yaml

@@ -2225,6 +2225,28 @@ spec:
                       must be reachable from the host where the installer is run.
                       Default is qemu:///system
                     type: string
+                  loadBalancer:
+                    description: LoadBalancer defines how the load balancer used by
+                      the cluster is configured. LoadBalancer is available in TechPreview.
+                    properties:
+                      type:
+                        default: OpenShiftManagedDefault
+                        description: type defines the type of load balancer used by
+                          the cluster on BareMetal platform which can be a user-managed
+                          or openshift-managed load balancer that is to be used for
+                          the OpenShift API and Ingress endpoints. When set to OpenShiftManagedDefault
+                          the static pods in charge of API and Ingress traffic load-balancing
+                          defined in the machine config operator will be deployed.
+                          When set to UserManaged these static pods will not be deployed
+                          and it is expected that the load balancer is configured
+                          out of band by the deployer. When omitted, this means no
+                          opinion and the platform is left to choose a reasonable
+                          default. The default value is OpenShiftManagedDefault.
+                        enum:
+                        - OpenShiftManagedDefault
+                        - UserManaged
+                        type: string
+                    type: object
                   provisioningBridge:
                     description: Provisioning bridge is used for provisioning nodes,
                       on the host that will run the bootstrap VM.
@@ -2627,6 +2649,28 @@ spec:
                     maxItems: 2
                     type: array
                     uniqueItems: true
+                  loadBalancer:
+                    description: LoadBalancer defines how the load balancer used by
+                      the cluster is configured. LoadBalancer is available in TechPreview.
+                    properties:
+                      type:
+                        default: OpenShiftManagedDefault
+                        description: type defines the type of load balancer used by
+                          the cluster on Nutanix platform which can be a user-managed
+                          or openshift-managed load balancer that is to be used for
+                          the OpenShift API and Ingress endpoints. When set to OpenShiftManagedDefault
+                          the static pods in charge of API and Ingress traffic load-balancing
+                          defined in the machine config operator will be deployed.
+                          When set to UserManaged these static pods will not be deployed
+                          and it is expected that the load balancer is configured
+                          out of band by the deployer. When omitted, this means no
+                          opinion and the platform is left to choose a reasonable
+                          default. The default value is OpenShiftManagedDefault.
+                        enum:
+                        - OpenShiftManagedDefault
+                        - UserManaged
+                        type: string
+                    type: object
                   prismCentral:
                     description: PrismCentral is the endpoint (address and port) and
                       credentials to connect to the Prism Central.
@@ -2871,6 +2915,28 @@ spec:
                       IP in your OpenStack cluster to associate with the OpenShift
                       load balancer. Deprecated: this value has been renamed to apiFloatingIP.'
                     type: string
+                  loadBalancer:
+                    description: LoadBalancer defines how the load balancer used by
+                      the cluster is configured. LoadBalancer is available in TechPreview.
+                    properties:
+                      type:
+                        default: OpenShiftManagedDefault
+                        description: type defines the type of load balancer used by
+                          the cluster on OpenStack platform which can be a user-managed
+                          or openshift-managed load balancer that is to be used for
+                          the OpenShift API and Ingress endpoints. When set to OpenShiftManagedDefault
+                          the static pods in charge of API and Ingress traffic load-balancing
+                          defined in the machine config operator will be deployed.
+                          When set to UserManaged these static pods will not be deployed
+                          and it is expected that the load balancer is configured
+                          out of band by the deployer. When omitted, this means no
+                          opinion and the platform is left to choose a reasonable
+                          default. The default value is OpenShiftManagedDefault.
+                        enum:
+                        - OpenShiftManagedDefault
+                        - UserManaged
+                        type: string
+                    type: object
                   machinesSubnet:
                     description: MachinesSubnet is the UUIDv4 of an openstack subnet.
                       This subnet will be used by all nodes created by the installer.
@@ -3065,6 +3131,28 @@ spec:
                     maxItems: 2
                     type: array
                     uniqueItems: true
+                  loadBalancer:
+                    description: LoadBalancer defines how the load balancer used by
+                      the cluster is configured. LoadBalancer is available in TechPreview.
+                    properties:
+                      type:
+                        default: OpenShiftManagedDefault
+                        description: type defines the type of load balancer used by
+                          the cluster on Ovirt platform which can be a user-managed
+                          or openshift-managed load balancer that is to be used for
+                          the OpenShift API and Ingress endpoints. When set to OpenShiftManagedDefault
+                          the static pods in charge of API and Ingress traffic load-balancing
+                          defined in the machine config operator will be deployed.
+                          When set to UserManaged these static pods will not be deployed
+                          and it is expected that the load balancer is configured
+                          out of band by the deployer. When omitted, this means no
+                          opinion and the platform is left to choose a reasonable
+                          default. The default value is OpenShiftManagedDefault.
+                        enum:
+                        - OpenShiftManagedDefault
+                        - UserManaged
+                        type: string
+                    type: object
                   ovirt_cluster_id:
                     description: The target cluster under which all VMs will run
                     type: string
@@ -3372,6 +3460,28 @@ spec:
                     maxItems: 2
                     type: array
                     uniqueItems: true
+                  loadBalancer:
+                    description: LoadBalancer defines how the load balancer used by
+                      the cluster is configured. LoadBalancer is available in TechPreview.
+                    properties:
+                      type:
+                        default: OpenShiftManagedDefault
+                        description: type defines the type of load balancer used by
+                          the cluster on VSphere platform which can be a user-managed
+                          or openshift-managed load balancer that is to be used for
+                          the OpenShift API and Ingress endpoints. When set to OpenShiftManagedDefault
+                          the static pods in charge of API and Ingress traffic load-balancing
+                          defined in the machine config operator will be deployed.
+                          When set to UserManaged these static pods will not be deployed
+                          and it is expected that the load balancer is configured
+                          out of band by the deployer. When omitted, this means no
+                          opinion and the platform is left to choose a reasonable
+                          default. The default value is OpenShiftManagedDefault.
+                        enum:
+                        - OpenShiftManagedDefault
+                        - UserManaged
+                        type: string
+                    type: object
                   network:
                     description: 'Network specifies the name of the network to be
                       used by the cluster. Deprecated: Use FailureDomains.Topology.Network'

data/data/openstack/bootstrap/main.tf

@@ -52,8 +52,11 @@ resource "openstack_networking_port_v2" "bootstrap_port" {
     subnet_id = var.nodes_subnet_id
   }
 
-  allowed_address_pairs {
-    ip_address = var.openstack_api_int_ip
+  dynamic "allowed_address_pairs" {
+    for_each = var.openstack_user_managed_load_balancer ? [] : [1]
+    content {
+      ip_address = var.openstack_api_int_ip
+    }
   }
 
   depends_on = [var.master_port_ids]

data/data/openstack/masters/private-network.tf

@@ -62,18 +62,25 @@ resource "openstack_networking_port_v2" "masters" {
     subnet_id = local.nodes_subnet_id
   }
 
-  allowed_address_pairs {
-    ip_address = var.openstack_api_int_ip
+  dynamic "allowed_address_pairs" {
+    for_each = var.openstack_user_managed_load_balancer ? [] : [1]
+    content {
+      ip_address = var.openstack_api_int_ip
+    }
   }
 
-  allowed_address_pairs {
-    ip_address = var.openstack_ingress_ip
+  dynamic "allowed_address_pairs" {
+    for_each = var.openstack_user_managed_load_balancer ? [] : [1]
+    content {
+      ip_address = var.openstack_ingress_ip
+    }
   }
 
   depends_on = [openstack_networking_port_v2.api_port, openstack_networking_port_v2.ingress_port]
 }
 
 resource "openstack_networking_port_v2" "api_port" {
+  count       = var.openstack_user_managed_load_balancer ? 0 : 1
   name        = "${var.cluster_id}-api-port"
   description = local.description
 
@@ -89,6 +96,7 @@ resource "openstack_networking_port_v2" "api_port" {
 }
 
 resource "openstack_networking_port_v2" "ingress_port" {
+  count       = var.openstack_user_managed_load_balancer ? 0 : 1
   name        = "${var.cluster_id}-ingress-port"
   description = local.description
 
@@ -134,15 +142,15 @@ resource "openstack_networking_trunk_v2" "masters" {
 // as expected.
 
 resource "openstack_networking_floatingip_associate_v2" "api_fip" {
-  count       = length(var.openstack_api_floating_ip) == 0 ? 0 : 1
-  port_id     = openstack_networking_port_v2.api_port.id
+  count       = (var.openstack_user_managed_load_balancer || length(var.openstack_api_floating_ip) == 0) ? 0 : 1
+  port_id     = openstack_networking_port_v2.api_port[0].id
   floating_ip = var.openstack_api_floating_ip
   depends_on  = [openstack_networking_router_interface_v2.nodes_router_interface]
 }
 
 resource "openstack_networking_floatingip_associate_v2" "ingress_fip" {
-  count       = length(var.openstack_ingress_floating_ip) == 0 ? 0 : 1
-  port_id     = openstack_networking_port_v2.ingress_port.id
+  count       = (var.openstack_user_managed_load_balancer || length(var.openstack_ingress_floating_ip) == 0) ? 0 : 1
+  port_id     = openstack_networking_port_v2.ingress_port[0].id
   floating_ip = var.openstack_ingress_floating_ip
   depends_on  = [openstack_networking_router_interface_v2.nodes_router_interface]
 }

data/data/openstack/variables-openstack.tf

@@ -374,3 +374,8 @@ variable "openstack_worker_server_group_policy" {
   type        = string
   description = "Policy of the server groups for the worker nodes."
 }
+
+variable "openstack_user_managed_load_balancer" {
+  type        = bool
+  description = "True if the load balancer that is used for the control plane VIPs is managed by the user."
+}

pkg/asset/installconfig/ovirt/validation.go

@@ -27,7 +27,7 @@ func Validate(ic *types.InstallConfig) error {
 
 	allErrs = append(
 		allErrs,
-		validation.ValidatePlatform(ic.Platform.Ovirt, ovirtPlatformPath)...)
+		validation.ValidatePlatform(ic.Platform.Ovirt, ovirtPlatformPath, ic)...)
 
 	con, err := NewConnection()
 	if err != nil {

pkg/asset/installconfig/vsphere/validation.go

@@ -59,7 +59,7 @@ func Validate(ic *types.InstallConfig) error {
 	if ic.Platform.VSphere == nil {
 		return errors.New(field.Required(field.NewPath("platform", "vsphere"), "vSphere validation requires a vSphere platform configuration").Error())
 	}
-	return validation.ValidatePlatform(ic.Platform.VSphere, field.NewPath("platform").Child("vsphere")).ToAggregate()
+	return validation.ValidatePlatform(ic.Platform.VSphere, field.NewPath("platform").Child("vsphere"), ic).ToAggregate()
 }
 
 func getVCenterClient(failureDomain vsphere.FailureDomain, ic *types.InstallConfig) (*validationContext, ClientLogout, error) {

pkg/asset/manifests/infrastructure.go

@@ -160,6 +160,7 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 			IngressIP:            installConfig.Config.Platform.BareMetal.IngressVIPs[0],
 			APIServerInternalIPs: installConfig.Config.Platform.BareMetal.APIVIPs,
 			IngressIPs:           installConfig.Config.Platform.BareMetal.IngressVIPs,
+			LoadBalancer:         installConfig.Config.Platform.BareMetal.LoadBalancer,
 		}
 	case gcp.Name:
 		config.Spec.PlatformSpec.Type = configv1.GCPPlatformType
@@ -210,6 +211,7 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 			IngressIP:            installConfig.Config.OpenStack.IngressVIPs[0],
 			APIServerInternalIPs: installConfig.Config.OpenStack.APIVIPs,
 			IngressIPs:           installConfig.Config.OpenStack.IngressVIPs,
+			LoadBalancer:         installConfig.Config.OpenStack.LoadBalancer,
 		}
 	case vsphere.Name:
 		config.Spec.PlatformSpec.Type = configv1.VSpherePlatformType
@@ -219,8 +221,10 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 				IngressIP:            installConfig.Config.VSphere.IngressVIPs[0],
 				APIServerInternalIPs: installConfig.Config.VSphere.APIVIPs,
 				IngressIPs:           installConfig.Config.VSphere.IngressVIPs,
+				LoadBalancer:         installConfig.Config.VSphere.LoadBalancer,
 			}
 		}
+
 		config.Spec.PlatformSpec.VSphere = vsphereinfra.GetInfraPlatformSpec(installConfig)
 
 		if _, exists := cloudproviderconfig.ConfigMap.Data["vsphere.conf"]; exists {
@@ -234,6 +238,7 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 			IngressIP:            installConfig.Config.Ovirt.IngressVIPs[0],
 			APIServerInternalIPs: installConfig.Config.Ovirt.APIVIPs,
 			IngressIPs:           installConfig.Config.Ovirt.IngressVIPs,
+			LoadBalancer:         installConfig.Config.Ovirt.LoadBalancer,
 		}
 	case powervs.Name:
 		config.Spec.PlatformSpec.Type = configv1.PowerVSPlatformType
@@ -303,6 +308,7 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error {
 				IngressIP:            installConfig.Config.Nutanix.IngressVIPs[0],
 				APIServerInternalIPs: installConfig.Config.Nutanix.APIVIPs,
 				IngressIPs:           installConfig.Config.Nutanix.IngressVIPs,
+				LoadBalancer:         installConfig.Config.Nutanix.LoadBalancer,
 			}
 		}
 	default:

pkg/tfvars/openstack/openstack.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
 	"github.com/gophercloud/utils/openstack/clientconfig"
 
+	configv1 "github.com/openshift/api/config/v1"
 	machinev1alpha1 "github.com/openshift/api/machine/v1alpha1"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	installconfig_openstack "github.com/openshift/installer/pkg/asset/installconfig/openstack"
@@ -95,6 +96,11 @@ func TFVars(
 		workermpool = installConfig.Config.Compute[0].Platform.OpenStack
 	}
 
+	var userManagedLoadBalancer bool
+	if lb := installConfig.Config.Platform.OpenStack.LoadBalancer; lb != nil && lb.Type == configv1.LoadBalancerTypeUserManaged {
+		userManagedLoadBalancer = true
+	}
+
 	var zones []string
 	{
 		seen := make(map[string]struct{})
@@ -211,6 +217,7 @@ func TFVars(
 		MachinesNetwork                   string                            `json:"openstack_machines_network_id,omitempty"`
 		MasterAvailabilityZones           []string                          `json:"openstack_master_availability_zones,omitempty"`
 		MasterRootVolumeAvailabilityZones []string                          `json:"openstack_master_root_volume_availability_zones,omitempty"`
+		UserManagedLoadBalancer           bool                              `json:"openstack_user_managed_load_balancer"`
 	}{
 		BaseImageName:                     imageName,
 		ExternalNetwork:                   installConfig.Config.Platform.OpenStack.ExternalNetwork,
@@ -236,6 +243,7 @@ func TFVars(
 		MachinesNetwork:                   machinesNetwork,
 		MasterAvailabilityZones:           zones,
 		MasterRootVolumeAvailabilityZones: masterRootVolumeAvailabilityZones,
+		UserManagedLoadBalancer:           userManagedLoadBalancer,
 	}, "", "  ")
 }
 

pkg/types/baremetal/platform.go

@@ -3,6 +3,7 @@ package baremetal
 import (
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
+	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/ipnet"
 )
 
@@ -226,4 +227,9 @@ type Platform struct {
 	// +kubebuilder:validation:Format=ip
 	// +optional
 	BootstrapExternalStaticGateway string `json:"bootstrapExternalStaticGateway,omitempty"`
+
+	// LoadBalancer defines how the load balancer used by the cluster is configured.
+	// LoadBalancer is available in TechPreview.
+	// +optional
+	LoadBalancer *configv1.BareMetalPlatformLoadBalancer `json:"loadBalancer,omitempty"`
 }

pkg/types/baremetal/validation/platform.go

@@ -19,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 
+	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/ipnet"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/baremetal"
@@ -425,9 +426,32 @@ func ValidatePlatform(p *baremetal.Platform, n *types.Networking, fldPath *field
 		allErrs = append(allErrs, validateHostsName(p.Hosts, fldPath.Child("Hosts"))...)
 	}
 
+	// Platform fields only allowed in TechPreviewNoUpgrade
+	if c.FeatureSet != configv1.TechPreviewNoUpgrade {
+		if c.BareMetal.LoadBalancer != nil {
+			allErrs = append(allErrs, field.Forbidden(fldPath.Child("loadBalancer"), "load balancer is not supported in this feature set"))
+		}
+	}
+
+	if c.BareMetal.LoadBalancer != nil {
+		if !validateLoadBalancer(c.BareMetal.LoadBalancer.Type) {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("loadBalancer", "type"), c.BareMetal.LoadBalancer.Type, "invalid load balancer type"))
+		}
+	}
+
 	return allErrs
 }
 
+// validateLoadBalancer returns an error if the load balancer is not valid.
+func validateLoadBalancer(lbType configv1.PlatformLoadBalancerType) bool {
+	switch lbType {
+	case configv1.LoadBalancerTypeOpenShiftManagedDefault, configv1.LoadBalancerTypeUserManaged:
+		return true
+	default:
+		return false
+	}
+}
+
 // ValidateProvisioning checks that provisioning network requirements specified is valid.
 func ValidateProvisioning(p *baremetal.Platform, n *types.Networking, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}