From deb3a942bc228d01cddfd4e9fe01ca6f3936287c Mon Sep 17 00:00:00 2001
From: trown <trown@redhat.com>
Date: Fri, 12 Jul 2019 11:05:22 -0400
Subject: [PATCH] openstack: get ignition via IP

For platforms without a DNS as a Service available, there is a
chicken/egg scenario. This is because we cannot setup our own DNS
solution on the nodes before they are booted via ignition.

Instead, we get the ignition config via a virtual IP that is
conifgured and maintained by keepalived. The setup of keepalived
will be done via machine-config-operator, so it is not part of
this patch.

This patch only sets up the plumbing so that we can merge the
keepalived work while keeping CI green. Then we can put up a patch
that will switch to actually using this functionality.

This patch also adds an interface for other platforms that need this
functionality to work from. Specifically, these PRs can be rebased
on this work:
https://github.com/openshift/installer/pull/1873
https://github.com/openshift/installer/pull/1948
---
 data/data/openstack/main.tf                   |  4 +++
 data/data/openstack/masters/main.tf           |  2 ++
 data/data/openstack/masters/variables.tf      |  8 +++++
 .../openstack/topology/private-network.tf     | 16 ++++++++++
 data/data/openstack/topology/variables.tf     |  8 +++++
 data/data/openstack/variables-openstack.tf    | 10 +++++++
 pkg/asset/cluster/tfvars.go                   |  2 ++
 pkg/asset/ignition/machine/node.go            |  6 ++++
 pkg/asset/tls/mcscertkey.go                   |  4 +++
 pkg/tfvars/openstack/openstack.go             |  6 +++-
 pkg/types/defaults/installconfig.go           |  2 +-
 pkg/types/defaults/installconfig_test.go      |  2 +-
 pkg/types/openstack/defaults/platform.go      | 30 ++++++++++++++++++-
 pkg/types/openstack/platform.go               |  8 +++++
 14 files changed, 104 insertions(+), 4 deletions(-)

diff --git a/data/data/openstack/main.tf b/data/data/openstack/main.tf
index 2f4e1cb840c..12546955a26 100644
--- a/data/data/openstack/main.tf
+++ b/data/data/openstack/main.tf
@@ -67,6 +67,8 @@ module "masters" {
   master_port_names   = module.topology.master_port_names
   user_data_ign       = var.ignition_master
   service_vm_fixed_ip = module.topology.service_vm_fixed_ip
+  api_int_ip          = var.openstack_api_int_ip
+  node_dns_ip         = var.openstack_node_dns_ip
   master_sg_ids = concat(
     var.openstack_master_extra_sg_ids,
     [module.topology.master_sg_id],
@@ -84,6 +86,8 @@ module "topology" {
   external_network_id = var.openstack_external_network_id
   masters_count       = var.master_count
   lb_floating_ip      = var.openstack_lb_floating_ip
+  api_int_ip          = var.openstack_api_int_ip
+  node_dns_ip         = var.openstack_node_dns_ip
   trunk_support       = var.openstack_trunk_support
 }
 
diff --git a/data/data/openstack/masters/main.tf b/data/data/openstack/masters/main.tf
index 624ec41ee45..85ca44a707f 100644
--- a/data/data/openstack/masters/main.tf
+++ b/data/data/openstack/masters/main.tf
@@ -28,6 +28,8 @@ data "ignition_file" "clustervars" {
 
   content {
     content = <<EOF
+export API_VIP=${var.api_int_ip}
+export DNS_VIP=${var.node_dns_ip}
 export FLOATING_IP=${var.lb_floating_ip}
 export BOOTSTRAP_IP=${var.bootstrap_ip}
 ${replace(join("\n", formatlist("export MASTER_FIXED_IPS_%s=%s", var.master_port_names, var.master_ips)), "${var.cluster_id}-master-port-", "")}
diff --git a/data/data/openstack/masters/variables.tf b/data/data/openstack/masters/variables.tf
index c2fb2a3f88a..e0249b456a9 100644
--- a/data/data/openstack/masters/variables.tf
+++ b/data/data/openstack/masters/variables.tf
@@ -51,6 +51,14 @@ variable "user_data_ign" {
   type = string
 }
 
+variable "api_int_ip" {
+  type = string
+}
+
+variable "node_dns_ip" {
+  type = string
+}
+
 variable "service_vm_fixed_ip" {
   type = string
 }
diff --git a/data/data/openstack/topology/private-network.tf b/data/data/openstack/topology/private-network.tf
index 2b5053353e8..6002a892695 100644
--- a/data/data/openstack/topology/private-network.tf
+++ b/data/data/openstack/topology/private-network.tf
@@ -30,6 +30,14 @@ resource "openstack_networking_subnet_v2" "nodes" {
   network_id      = openstack_networking_network_v2.openshift-private.id
   tags            = ["openshiftClusterID=${var.cluster_id}"]
   dns_nameservers = [openstack_networking_port_v2.service_port.all_fixed_ips[0]]
+
+  # We reserve some space at the beginning of the CIDR to use for the VIPs
+  # It would be good to make this more dynamic by calculating the number of
+  # addresses in the provided CIDR. This currently assumes at least a /18.
+  allocation_pool {
+    start = cidrhost(local.nodes_cidr_block, 10)
+    end   = cidrhost(local.nodes_cidr_block, 16000)
+  }
 }
 
 resource "openstack_networking_port_v2" "masters" {
@@ -44,6 +52,14 @@ resource "openstack_networking_port_v2" "masters" {
   fixed_ip {
     subnet_id = openstack_networking_subnet_v2.nodes.id
   }
+
+  allowed_address_pairs {
+    ip_address = var.api_int_ip
+  }
+
+  allowed_address_pairs {
+    ip_address = var.node_dns_ip
+  }
 }
 
 resource "openstack_networking_trunk_v2" "masters" {
diff --git a/data/data/openstack/topology/variables.tf b/data/data/openstack/topology/variables.tf
index ffcec57f9b2..7272283efdf 100644
--- a/data/data/openstack/topology/variables.tf
+++ b/data/data/openstack/topology/variables.tf
@@ -28,6 +28,14 @@ variable "masters_count" {
   type = string
 }
 
+variable "api_int_ip" {
+  type = string
+}
+
+variable "node_dns_ip" {
+  type = string
+}
+
 variable "trunk_support" {
   type = string
 }
diff --git a/data/data/openstack/variables-openstack.tf b/data/data/openstack/variables-openstack.tf
index 517ee3ee8e2..c6437db91e6 100644
--- a/data/data/openstack/variables-openstack.tf
+++ b/data/data/openstack/variables-openstack.tf
@@ -269,6 +269,16 @@ EOF
 
 }
 
+variable "openstack_api_int_ip" {
+  type        = string
+  description = "IP on the node subnet reserved for api-int VIP."
+}
+
+variable "openstack_node_dns_ip" {
+  type        = string
+  description = "IP on the nodes subnet reserved for node dns VIP."
+}
+
 variable "openstack_master_flavor_name" {
   type        = string
   description = "Instance size for the master node(s). Example: `m1.medium`."
diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go
index d0090a4caf0..9d1c990514d 100644
--- a/pkg/asset/cluster/tfvars.go
+++ b/pkg/asset/cluster/tfvars.go
@@ -248,6 +248,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			installConfig.Config.Platform.OpenStack.Region,
 			installConfig.Config.Platform.OpenStack.ExternalNetwork,
 			installConfig.Config.Platform.OpenStack.LbFloatingIP,
+			installConfig.Config.Platform.OpenStack.APIVIP,
+			installConfig.Config.Platform.OpenStack.DNSVIP,
 			installConfig.Config.Platform.OpenStack.TrunkSupport,
 		)
 		if err != nil {
diff --git a/pkg/asset/ignition/machine/node.go b/pkg/asset/ignition/machine/node.go
index d1930d25559..681851c5c40 100644
--- a/pkg/asset/ignition/machine/node.go
+++ b/pkg/asset/ignition/machine/node.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/openshift/installer/pkg/types"
 	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 )
 
 // pointerIgnitionConfig generates a config which references the remote config
@@ -20,6 +21,11 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
 		// Baremetal needs to point directly at the VIP because we don't have a
 		// way to configure DNS before Ignition runs.
 		ignitionHost = fmt.Sprintf("%s:22623", installConfig.BareMetal.APIVIP)
+	case openstacktypes.Name:
+		// We can't actually set this to the VIP until we have keepalived patches
+		// merged to machine-config-operator
+		// ignitionHost = fmt.Sprintf("%s:22623", installConfig.Config.OpenStack.APIVIP)
+		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 	default:
 		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 	}
diff --git a/pkg/asset/tls/mcscertkey.go b/pkg/asset/tls/mcscertkey.go
index cd0a5c79b9f..161d0fe0bbc 100644
--- a/pkg/asset/tls/mcscertkey.go
+++ b/pkg/asset/tls/mcscertkey.go
@@ -8,6 +8,7 @@ import (
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 )
 
 // MCSCertKey is the asset that generates the MCS key/cert pair.
@@ -45,6 +46,9 @@ func (a *MCSCertKey) Generate(dependencies asset.Parents) error {
 	case baremetaltypes.Name:
 		cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.BareMetal.APIVIP)}
 		cfg.DNSNames = []string{hostname, installConfig.Config.BareMetal.APIVIP}
+	case openstacktypes.Name:
+		cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.OpenStack.APIVIP)}
+		cfg.DNSNames = []string{hostname, installConfig.Config.OpenStack.APIVIP}
 	default:
 		cfg.DNSNames = []string{hostname}
 	}
diff --git a/pkg/tfvars/openstack/openstack.go b/pkg/tfvars/openstack/openstack.go
index 4971a372535..444f39d76f8 100644
--- a/pkg/tfvars/openstack/openstack.go
+++ b/pkg/tfvars/openstack/openstack.go
@@ -14,11 +14,13 @@ type config struct {
 	Cloud           string `json:"openstack_credentials_cloud,omitempty"`
 	FlavorName      string `json:"openstack_master_flavor_name,omitempty"`
 	LbFloatingIP    string `json:"openstack_lb_floating_ip,omitempty"`
+	APIVIP          string `json:"openstack_api_int_ip,omitempty"`
+	DNSVIP          string `json:"openstack_node_dns_ip,omitempty"`
 	TrunkSupport    string `json:"openstack_trunk_support,omitempty"`
 }
 
 // TFVars generates OpenStack-specific Terraform variables.
-func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externalNetwork string, lbFloatingIP string, trunkSupport string) ([]byte, error) {
+func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externalNetwork string, lbFloatingIP string, apiVIP string, dnsVIP string, trunkSupport string) ([]byte, error) {
 	cfg := &config{
 		Region:          region,
 		BaseImage:       masterConfig.Image,
@@ -26,6 +28,8 @@ func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externa
 		Cloud:           masterConfig.CloudName,
 		FlavorName:      masterConfig.Flavor,
 		LbFloatingIP:    lbFloatingIP,
+		APIVIP:          apiVIP,
+		DNSVIP:          dnsVIP,
 		TrunkSupport:    trunkSupport,
 	}
 
diff --git a/pkg/types/defaults/installconfig.go b/pkg/types/defaults/installconfig.go
index 879fec46ac8..89e82e90d23 100644
--- a/pkg/types/defaults/installconfig.go
+++ b/pkg/types/defaults/installconfig.go
@@ -67,7 +67,7 @@ func SetInstallConfigDefaults(c *types.InstallConfig) {
 	case c.Platform.Libvirt != nil:
 		libvirtdefaults.SetPlatformDefaults(c.Platform.Libvirt)
 	case c.Platform.OpenStack != nil:
-		openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack)
+		openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack, c)
 	case c.Platform.VSphere != nil:
 		vspheredefaults.SetPlatformDefaults(c.Platform.VSphere, c)
 	case c.Platform.BareMetal != nil:
diff --git a/pkg/types/defaults/installconfig_test.go b/pkg/types/defaults/installconfig_test.go
index 9f449a6a513..1961fcd1a3e 100644
--- a/pkg/types/defaults/installconfig_test.go
+++ b/pkg/types/defaults/installconfig_test.go
@@ -65,7 +65,7 @@ func defaultLibvirtInstallConfig() *types.InstallConfig {
 func defaultOpenStackInstallConfig() *types.InstallConfig {
 	c := defaultInstallConfig()
 	c.Platform.OpenStack = &openstack.Platform{}
-	openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack)
+	openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack, c)
 	return c
 }
 
diff --git a/pkg/types/openstack/defaults/platform.go b/pkg/types/openstack/defaults/platform.go
index b7817671302..f7b72f7ecc6 100644
--- a/pkg/types/openstack/defaults/platform.go
+++ b/pkg/types/openstack/defaults/platform.go
@@ -1,9 +1,37 @@
 package defaults
 
 import (
+	"fmt"
+
+	"github.com/apparentlymart/go-cidr/cidr"
+
+	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 )
 
+// Defaults for the openstack platform.
+const (
+	APIVIP = ""
+	DNSVIP = ""
+)
+
 // SetPlatformDefaults sets the defaults for the platform.
-func SetPlatformDefaults(p *openstack.Platform) {
+func SetPlatformDefaults(p *openstack.Platform, c *types.InstallConfig) {
+	if p.APIVIP == "" {
+		vip, err := cidr.Host(&c.Networking.MachineCIDR.IPNet, 5)
+		if err != nil {
+			p.APIVIP = fmt.Sprintf("failed to get APIVIP from MachineCIDR: %s", err.Error())
+		} else {
+			p.APIVIP = vip.String()
+		}
+	}
+
+	if p.DNSVIP == "" {
+		vip, err := cidr.Host(&c.Networking.MachineCIDR.IPNet, 6)
+		if err != nil {
+			p.DNSVIP = fmt.Sprintf("failed to get DNSVIP from MachineCIDR: %s", err.Error())
+		} else {
+			p.DNSVIP = vip.String()
+		}
+	}
 }
diff --git a/pkg/types/openstack/platform.go b/pkg/types/openstack/platform.go
index 2308f0e50b9..3e0b0c55665 100644
--- a/pkg/types/openstack/platform.go
+++ b/pkg/types/openstack/platform.go
@@ -28,6 +28,14 @@ type Platform struct {
 	// Existing Floating IP to associate with the OpenStack load balancer.
 	LbFloatingIP string `json:"lbFloatingIP"`
 
+	// APIVIP
+	// IP in the machineCIDR to use for api-int.
+	APIVIP string `json:"apiVIP"`
+
+	// DNSVIP
+	// IP in the machineCIDR to use for simulated route53.
+	DNSVIP string `json:"dnsVIP"`
+
 	// TrunkSupport
 	// Whether OpenStack ports can be trunked
 	TrunkSupport string `json:"trunkSupport"`