From 72fb94b5a22184c6c345ffd6b00850f0d40d209b Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Tue, 10 Jan 2023 11:16:01 -0500 Subject: [PATCH 1/6] openstack: add condition to configure VIPs in PlatformStatus When the VIP parameters within the Infrastructure Status object are empty, MCO will not deploy the internal LB for on-prem (HAproxy/Keepalived). This allows to use an external Load-Balancer instead of the internal one. --- pkg/asset/manifests/infrastructure.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/asset/manifests/infrastructure.go b/pkg/asset/manifests/infrastructure.go index d92eff3e8ff..5f937d5b180 100644 --- a/pkg/asset/manifests/infrastructure.go +++ b/pkg/asset/manifests/infrastructure.go @@ -197,11 +197,13 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error { config.Spec.PlatformSpec.Type = configv1.NonePlatformType case openstack.Name: config.Spec.PlatformSpec.Type = configv1.OpenStackPlatformType - config.Status.PlatformStatus.OpenStack = &configv1.OpenStackPlatformStatus{ - APIServerInternalIP: installConfig.Config.OpenStack.APIVIPs[0], - IngressIP: installConfig.Config.OpenStack.IngressVIPs[0], - APIServerInternalIPs: installConfig.Config.OpenStack.APIVIPs, - IngressIPs: installConfig.Config.OpenStack.IngressVIPs, + if len(installConfig.Config.OpenStack.APIVIPs) > 0 { + config.Status.PlatformStatus.OpenStack = &configv1.OpenStackPlatformStatus{ + APIServerInternalIP: installConfig.Config.OpenStack.APIVIPs[0], + IngressIP: installConfig.Config.OpenStack.IngressVIPs[0], + APIServerInternalIPs: installConfig.Config.OpenStack.APIVIPs, + IngressIPs: installConfig.Config.OpenStack.IngressVIPs, + } } case vsphere.Name: config.Spec.PlatformSpec.Type = configv1.VSpherePlatformType From 44e7d38830682e08860ce076fdcd2f952f2aad36 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Tue, 10 Jan 2023 14:48:43 -0500 Subject: [PATCH 2/6] openstack: validate dns and external load balancer This PR adds validation to the OpenStack installation for DNS and LB existence. It will query DNS for api and api-int. If either fail a error will be provided. It will perform a TCP/6443 port connection to the api.ClusterDomain(). If not available will emit a warning only. We are re-using the function from vsphere/validation (we had to make it public). --- pkg/asset/installconfig/openstack/validate.go | 15 +++++++++++++++ pkg/asset/installconfig/vsphere/validation.go | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/asset/installconfig/openstack/validate.go b/pkg/asset/installconfig/openstack/validate.go index 9241f79e965..168d78e8e0c 100644 --- a/pkg/asset/installconfig/openstack/validate.go +++ b/pkg/asset/installconfig/openstack/validate.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "github.com/openshift/installer/pkg/asset/installconfig/openstack/validation" + vspherevalidation "github.com/openshift/installer/pkg/asset/installconfig/vsphere" "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/openstack" openstackdefaults "github.com/openshift/installer/pkg/types/openstack/defaults" @@ -59,6 +60,20 @@ func Validate(ic *types.InstallConfig) error { allErrs = append(allErrs, validation.ValidateMachinePool(&compute, ci, false, fldPath.Child("platform", "openstack"))...) } + // If APIVIPs and IngressVIPs is equal to zero + // then don't validate the VIPs. + // Instead, ensure there is a configured + // DNS record for api and test if the load + // balancer is configured. + // The VIP parameters within the Infrastructure status object + // will be empty. This will cause MCO to not deploy + // the static pods: haproxy, keepalived and coredns. + // This will allow the use of an external load balancer + // and RHCOS nodes to be on multiple L2 segments. + if len(ic.Platform.OpenStack.APIVIPs) == 0 && len(ic.Platform.OpenStack.IngressVIPs) == 0 { + allErrs = append(allErrs, vspherevalidation.EnsureLoadBalancerDNS(ic, field.NewPath("platform"))...) + } + return allErrs.ToAggregate() } diff --git a/pkg/asset/installconfig/vsphere/validation.go b/pkg/asset/installconfig/vsphere/validation.go index a40c8d4e327..f7e18ac9262 100644 --- a/pkg/asset/installconfig/vsphere/validation.go +++ b/pkg/asset/installconfig/vsphere/validation.go @@ -84,7 +84,7 @@ func ValidateMultiZoneForProvisioning(ic *types.InstallConfig) error { // This will allow the use of an external load balancer // and RHCOS nodes to be on multiple L2 segments. if len(ic.Platform.VSphere.APIVIPs) == 0 && len(ic.Platform.VSphere.IngressVIPs) == 0 { - allErrs = append(allErrs, ensureLoadBalancerDNS(ic, field.NewPath("platform"))...) + allErrs = append(allErrs, EnsureLoadBalancerDNS(ic, field.NewPath("platform"))...) } var clients = make(map[string]*validationContext, 0) @@ -427,7 +427,7 @@ func validateVcenterPrivileges(validationCtx *validationContext, fldPath *field. return field.ErrorList{} } -func ensureLoadBalancerDNS(installConfig *types.InstallConfig, fldPath *field.Path) field.ErrorList { +func EnsureLoadBalancerDNS(installConfig *types.InstallConfig, fldPath *field.Path) field.ErrorList { var lastErr error var uris []string dialTimeout := time.Second From cda5b9538514e65617955f936ae11d405280ffb6 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Tue, 10 Jan 2023 15:14:22 -0500 Subject: [PATCH 3/6] openstack: VIPs not required anymore in ic validation --- pkg/types/validation/installconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go index d051e1e93d5..1b5c3a1da09 100644 --- a/pkg/types/validation/installconfig.go +++ b/pkg/types/validation/installconfig.go @@ -503,7 +503,7 @@ func validateVIPsForPlatform(network *types.Networking, platform *types.Platform Ingress: platform.OpenStack.IngressVIPs, } - allErrs = append(allErrs, validateAPIAndIngressVIPs(virtualIPs, newVIPsFields, true, network, fldPath.Child(openstack.Name))...) + allErrs = append(allErrs, validateAPIAndIngressVIPs(virtualIPs, newVIPsFields, false, network, fldPath.Child(openstack.Name))...) case platform.VSphere != nil: allErrs = append(allErrs, ensureIPv4IsFirstInDualStackSlice(&platform.VSphere.APIVIPs, fldPath.Child(vsphere.Name, newVIPsFields.APIVIPs))...) allErrs = append(allErrs, ensureIPv4IsFirstInDualStackSlice(&platform.VSphere.IngressVIPs, fldPath.Child(vsphere.Name, newVIPsFields.IngressVIPs))...) From 45409b6407324944d227111a839ff38f8ac8d66f Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 11 Jan 2023 09:17:34 -0500 Subject: [PATCH 4/6] openstack: disable defaults for APIVIPs and IngressVIPs --- pkg/types/openstack/defaults/platform.go | 31 ------------------------ 1 file changed, 31 deletions(-) diff --git a/pkg/types/openstack/defaults/platform.go b/pkg/types/openstack/defaults/platform.go index d9c31c0b8e2..b2ae9e2e26d 100644 --- a/pkg/types/openstack/defaults/platform.go +++ b/pkg/types/openstack/defaults/platform.go @@ -1,11 +1,8 @@ package defaults import ( - "fmt" "os" - "github.com/apparentlymart/go-cidr/cidr" - "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/openstack" ) @@ -23,32 +20,4 @@ func SetPlatformDefaults(p *openstack.Platform, n *types.Networking) { p.Cloud = DefaultCloudName } } - // APIVIP returns the internal virtual IP address (VIP) put in front - // of the Kubernetes API server for use by components inside the - // cluster. The DNS static pods running on the nodes resolve the - // api-int record to APIVIP. - if len(p.APIVIPs) == 0 && p.DeprecatedAPIVIP == "" { - vip, err := cidr.Host(&n.MachineNetwork[0].CIDR.IPNet, 5) - if err != nil { - // This will fail validation and abort the install - p.APIVIPs = []string{fmt.Sprintf("could not derive API VIP from machine networks: %s", err.Error())} - } else { - p.APIVIPs = []string{vip.String()} - } - } - - // IngressVIP returns the internal virtual IP address (VIP) put in - // front of the OpenShift router pods. This provides the internal - // accessibility to the internal pods running on the worker nodes, - // e.g. `console`. The DNS static pods running on the nodes resolve - // the wildcard apps record to IngressVIP. - if len(p.IngressVIPs) == 0 && p.DeprecatedIngressVIP == "" { - vip, err := cidr.Host(&n.MachineNetwork[0].CIDR.IPNet, 7) - if err != nil { - // This will fail validation and abort the install - p.IngressVIPs = []string{fmt.Sprintf("could not derive Ingress VIP from machine networks: %s", err.Error())} - } else { - p.IngressVIPs = []string{vip.String()} - } - } } From 76911c3a21e8b1a492ac017d6bae2bd77690f8d5 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 11 Jan 2023 09:25:05 -0500 Subject: [PATCH 5/6] openstack: ingore empty APIVIPs for ignition config --- pkg/asset/ignition/machine/node.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/asset/ignition/machine/node.go b/pkg/asset/ignition/machine/node.go index 654d1c7ebed..243a4d59dee 100644 --- a/pkg/asset/ignition/machine/node.go +++ b/pkg/asset/ignition/machine/node.go @@ -39,7 +39,9 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro ignitionHost = net.JoinHostPort(installConfig.Nutanix.APIVIPs[0], "22623") } case openstacktypes.Name: - ignitionHost = net.JoinHostPort(installConfig.OpenStack.APIVIPs[0], "22623") + if len(installConfig.OpenStack.APIVIPs) > 0 { + ignitionHost = net.JoinHostPort(installConfig.OpenStack.APIVIPs[0], "22623") + } case ovirttypes.Name: ignitionHost = net.JoinHostPort(installConfig.Ovirt.APIVIPs[0], "22623") case vspheretypes.Name: From eaf0da69e0246497d46a312159d7102730bc8358 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 11 Jan 2023 09:39:37 -0500 Subject: [PATCH 6/6] openstack: adapt terraform to support external LB --- data/data/openstack/bootstrap/main.tf | 7 ++++-- .../data/openstack/masters/private-network.tf | 24 ++++++++++++------- data/data/openstack/variables-openstack.tf | 2 ++ pkg/tfvars/openstack/openstack.go | 10 ++++++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/data/data/openstack/bootstrap/main.tf b/data/data/openstack/bootstrap/main.tf index 667a2848e4e..21088c8f449 100644 --- a/data/data/openstack/bootstrap/main.tf +++ b/data/data/openstack/bootstrap/main.tf @@ -52,8 +52,11 @@ resource "openstack_networking_port_v2" "bootstrap_port" { subnet_id = var.nodes_subnet_id } - allowed_address_pairs { - ip_address = var.openstack_api_int_ip + dynamic "allowed_address_pairs" { + for_each = var.openstack_api_int_ip == "" ? []: [1] + content { + ip_address = var.openstack_api_int_ip + } } depends_on = [var.master_port_ids] diff --git a/data/data/openstack/masters/private-network.tf b/data/data/openstack/masters/private-network.tf index 2e108673f46..97d04276131 100644 --- a/data/data/openstack/masters/private-network.tf +++ b/data/data/openstack/masters/private-network.tf @@ -62,18 +62,25 @@ resource "openstack_networking_port_v2" "masters" { subnet_id = local.nodes_subnet_id } - allowed_address_pairs { - ip_address = var.openstack_api_int_ip + dynamic "allowed_address_pairs" { + for_each = var.openstack_api_int_ip == "" ? []: [1] + content { + ip_address = var.openstack_api_int_ip + } } - allowed_address_pairs { - ip_address = var.openstack_ingress_ip + dynamic "allowed_address_pairs" { + for_each = var.openstack_ingress_ip == "" ? []: [1] + content { + ip_address = var.openstack_ingress_ip + } } depends_on = [openstack_networking_port_v2.api_port, openstack_networking_port_v2.ingress_port] } resource "openstack_networking_port_v2" "api_port" { + count = var.openstack_api_int_ip == "" ? 0 : 1 name = "${var.cluster_id}-api-port" description = local.description @@ -89,6 +96,7 @@ resource "openstack_networking_port_v2" "api_port" { } resource "openstack_networking_port_v2" "ingress_port" { + count = var.openstack_ingress_ip == "" ? 0 : 1 name = "${var.cluster_id}-ingress-port" description = local.description @@ -134,15 +142,15 @@ resource "openstack_networking_trunk_v2" "masters" { // as expected. resource "openstack_networking_floatingip_associate_v2" "api_fip" { - count = length(var.openstack_api_floating_ip) == 0 ? 0 : 1 - port_id = openstack_networking_port_v2.api_port.id + count = (var.openstack_api_int_ip == "" || length(var.openstack_api_floating_ip) == 0) ? 0 : 1 + port_id = openstack_networking_port_v2.api_port[0].id floating_ip = var.openstack_api_floating_ip depends_on = [openstack_networking_router_interface_v2.nodes_router_interface] } resource "openstack_networking_floatingip_associate_v2" "ingress_fip" { - count = length(var.openstack_ingress_floating_ip) == 0 ? 0 : 1 - port_id = openstack_networking_port_v2.ingress_port.id + count = (var.openstack_ingress_ip == "" || length(var.openstack_ingress_floating_ip) == 0) ? 0 : 1 + port_id = openstack_networking_port_v2.ingress_port[0].id floating_ip = var.openstack_ingress_floating_ip depends_on = [openstack_networking_router_interface_v2.nodes_router_interface] } diff --git a/data/data/openstack/variables-openstack.tf b/data/data/openstack/variables-openstack.tf index 20b52125c9c..025f4860ef2 100644 --- a/data/data/openstack/variables-openstack.tf +++ b/data/data/openstack/variables-openstack.tf @@ -287,11 +287,13 @@ EOF variable "openstack_api_int_ip" { type = string description = "IP on the node subnet reserved for api-int VIP." + default = "" } variable "openstack_ingress_ip" { type = string description = "IP on the nodes subnet reserved for the ingress VIP." + default = "" } variable "openstack_external_dns" { diff --git a/pkg/tfvars/openstack/openstack.go b/pkg/tfvars/openstack/openstack.go index c2e78c28afd..e74b9aa8d37 100644 --- a/pkg/tfvars/openstack/openstack.go +++ b/pkg/tfvars/openstack/openstack.go @@ -186,6 +186,12 @@ func TFVars( } } + var apiVIP, ingressVIP string + if len(installConfig.Config.Platform.OpenStack.APIVIPs) > 0 { + apiVIP = installConfig.Config.Platform.OpenStack.APIVIPs[0] + ingressVIP = installConfig.Config.Platform.OpenStack.IngressVIPs[0] + } + return json.MarshalIndent(struct { BaseImageName string `json:"openstack_base_image_name,omitempty"` ExternalNetwork string `json:"openstack_external_network,omitempty"` @@ -218,8 +224,8 @@ func TFVars( FlavorName: masterSpecs[0].Flavor, APIFloatingIP: installConfig.Config.Platform.OpenStack.APIFloatingIP, IngressFloatingIP: installConfig.Config.Platform.OpenStack.IngressFloatingIP, - APIVIP: installConfig.Config.Platform.OpenStack.APIVIPs[0], - IngressVIP: installConfig.Config.Platform.OpenStack.IngressVIPs[0], + APIVIP: apiVIP, + IngressVIP: ingressVIP, TrunkSupport: masterSpecs[0].Trunk, OctaviaSupport: octaviaSupport, RootVolumeSize: rootVolumeSize,