From 419269955eccb5464f8b5e2a8e9fe5fe9a8c35ef Mon Sep 17 00:00:00 2001
From: Petr Horacek <phoracek@redhat.com>
Date: Thu, 29 Aug 2019 00:40:58 +0200
Subject: [PATCH 1/2] configure host networking policy with default bridge

Configure linux bridge as the default network interface.

Signed-off-by: Petr Horacek <phoracek@redhat.com>
---
 CNV/Makefile                                  |  1 +
 CNV/cnv-2.1.0.sh                              |  3 ++
 CNV/configure-network.sh                      | 53 +++++++++++++++++++
 OpenShift/99_post_install.sh                  | 25 ---------
 .../99-brext-master.yaml.template             | 22 --------
 .../assets/post-install/ifcfg-bridge.template |  8 ---
 .../post-install/ifcfg-interface.template     |  5 --
 7 files changed, 57 insertions(+), 60 deletions(-)
 create mode 100755 CNV/configure-network.sh
 delete mode 100644 OpenShift/assets/post-install/99-brext-master.yaml.template
 delete mode 100644 OpenShift/assets/post-install/ifcfg-bridge.template
 delete mode 100644 OpenShift/assets/post-install/ifcfg-interface.template

diff --git a/CNV/Makefile b/CNV/Makefile
index c17958e..4ab34f8 100644
--- a/CNV/Makefile
+++ b/CNV/Makefile
@@ -6,6 +6,7 @@ all: deploy
 
 deploy:
 	./cnv-2.1.0.sh
+	./configure-network.sh
 
 upgrade:
 	./cnv-upgrade.sh
diff --git a/CNV/cnv-2.1.0.sh b/CNV/cnv-2.1.0.sh
index 3432fae..617f2fe 100755
--- a/CNV/cnv-2.1.0.sh
+++ b/CNV/cnv-2.1.0.sh
@@ -162,3 +162,6 @@ metadata:
 spec:
   BareMetalPlatform: true
 EOF
+
+echo "Waiting for HCO to get fully deployed"
+oc wait -n ${TARGET_NAMESPACE} hyperconverged hyperconverged-cluster --for condition=Available --timeout=10m
diff --git a/CNV/configure-network.sh b/CNV/configure-network.sh
new file mode 100755
index 0000000..2181475
--- /dev/null
+++ b/CNV/configure-network.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -ex
+
+MACHINE_CIDR=$(grep 'machineCIDR' ../OpenShift/install-config.yaml | sed 's/\(.*\): *\(.*\)/\2/')
+BRIDGE_NAME=brext
+
+export KUBECONFIG=${KUBECONFIG:-../OpenShift/ocp/auth/kubeconfig}
+
+echo "Configuring networks on nodes"
+
+echo "Detecting the default interface"
+while ! default_iface=$(oc get nodenetworkstate ${node} -o jsonpath="{.items[0].status.currentState.routes.running[?(@.destination==\"${MACHINE_CIDR}\")].next-hop-interface}" | cut -d " " -f 1); do
+    sleep 10
+done
+
+if [ "${default_iface}" == "${BRIDGE_NAME}" ]; then
+    echo "Bridge ${BRIDGE_NAME} seems to be already configured as the default interface, skipping the rest of network setup"
+    exit 0
+fi
+
+echo "Applying node network configuration policy"
+cat <<EOF | oc apply -f -
+apiVersion: nmstate.io/v1alpha1
+kind: NodeNetworkConfigurationPolicy
+metadata:
+  name: kni-policy
+spec:
+  desiredState:
+    interfaces:
+    - name: ${BRIDGE_NAME}
+      type: linux-bridge
+      state: up
+      ipv4:
+        dhcp: true
+        enabled: true
+      ipv6:
+        dhcp: true
+        enabled: true
+      bridge:
+        options:
+          stp:
+            enabled: false
+        port:
+        - name: ${default_iface}
+EOF
+
+echo "Waiting until the configuration is done, it may take up to 5 minutes until keepalived gets reconfigured"
+nodes=$(oc get nodes -o jsonpath='{range .items[*]}{.metadata.name} {end}')
+for node in $nodes; do
+    until [ "$(oc get nodenetworkstate ${node} -o jsonpath="{.status.currentState.routes.running[?(@.destination==\"${MACHINE_CIDR}\")].next-hop-interface}")" == "${BRIDGE_NAME}" ]; do sleep 10; done
+done
+oc wait node --all --for condition=Ready --timeout=10m
diff --git a/OpenShift/99_post_install.sh b/OpenShift/99_post_install.sh
index e61d8d1..287b52e 100755
--- a/OpenShift/99_post_install.sh
+++ b/OpenShift/99_post_install.sh
@@ -8,32 +8,8 @@ source common.sh
 export KUBECONFIG=${KUBECONFIG:-ocp/auth/kubeconfig}
 POSTINSTALL_ASSETS_DIR="./assets/post-install"
 IFCFG_INTERFACE="${POSTINSTALL_ASSETS_DIR}/ifcfg-interface.template"
-IFCFG_BRIDGE="${POSTINSTALL_ASSETS_DIR}/ifcfg-bridge.template"
-BREXT_FILE="${POSTINSTALL_ASSETS_DIR}/99-brext-master.yaml"
 MACHINE_DATA_PATCH_DIR="../preflight"
 
-export bridge="${bridge:-brext}"
-
-create_bridge(){
-  echo "Deploying Bridge ${bridge}..."
-
-  FIRST_MASTER=$(oc get node -o custom-columns=IP:.status.addresses[0].address --no-headers | head -1)
-  export interface=$(ssh -q -o StrictHostKeyChecking=no core@$FIRST_MASTER "ip r | grep default | grep -Po  '(?<=dev )(\S+)'")
-  if [ "$interface" == "" ] ; then
-    echo "Issue detecting interface to use! Leaving..."
-    exit 1
-  fi
-  if [ "$interface" != "$bridge" ] ; then
-    echo "Using interface $interface"
-    export interface_content=$(envsubst < ${IFCFG_INTERFACE} | base64 -w0)
-    export bridge_content=$(envsubst < ${IFCFG_BRIDGE} | base64 -w0)
-    envsubst < ${BREXT_FILE}.template > ${BREXT_FILE}
-    echo "Done creating bridge definition"
-  else
-    echo "Bridge already there!"
-  fi
-}
-
 apply_mc(){
   # Disable auto reboot hosts in order to apply several mcos at the same time
   for node_type in master worker; do
@@ -229,6 +205,5 @@ function add-machine-ips() {
 }
 
 add-machine-ips
-create_bridge
 create_ntp_config
 apply_mc
diff --git a/OpenShift/assets/post-install/99-brext-master.yaml.template b/OpenShift/assets/post-install/99-brext-master.yaml.template
deleted file mode 100644
index d0e6225..0000000
--- a/OpenShift/assets/post-install/99-brext-master.yaml.template
+++ /dev/null
@@ -1,22 +0,0 @@
-apiVersion: machineconfiguration.openshift.io/v1
-kind: MachineConfig
-metadata:
-  labels:
-    machineconfiguration.openshift.io/role: master
-  name: 99-brext-master
-spec:
-  config:
-    ignition:
-      version: 2.2.0
-    storage:
-      files:
-      - contents:
-          source: data:text/plain;charset=utf-8;base64,${interface_content}
-        filesystem: root
-        mode: 0644
-        path: /etc/sysconfig/network-scripts/ifcfg-${interface}
-      - contents:
-          source: data:text/plain;charset=utf-8;base64,${bridge_content}
-        filesystem: root
-        mode: 0644
-        path: /etc/sysconfig/network-scripts/ifcfg-${bridge}
diff --git a/OpenShift/assets/post-install/ifcfg-bridge.template b/OpenShift/assets/post-install/ifcfg-bridge.template
deleted file mode 100644
index ac4de35..0000000
--- a/OpenShift/assets/post-install/ifcfg-bridge.template
+++ /dev/null
@@ -1,8 +0,0 @@
-DEVICE=${bridge}
-NAME=${bridge}
-TYPE=Bridge
-ONBOOT=yes
-NM_CONTROLLED=yes
-BOOTPROTO=dhcp
-BRIDGING_OPTS=vlan_filtering=1
-BRIDGE_VLANS="1 pvid untagged,20,300-400 untagged"
diff --git a/OpenShift/assets/post-install/ifcfg-interface.template b/OpenShift/assets/post-install/ifcfg-interface.template
deleted file mode 100644
index c010bc6..0000000
--- a/OpenShift/assets/post-install/ifcfg-interface.template
+++ /dev/null
@@ -1,5 +0,0 @@
-DEVICE=${interface}
-BRIDGE=${bridge}
-ONBOOT=yes
-NM_CONTROLLED=yes
-BOOTPROTO=none

From a54fe41419406b22d8f276d2e3a08732101d48a9 Mon Sep 17 00:00:00 2001
From: Petr Horacek <phoracek@redhat.com>
Date: Fri, 20 Sep 2019 12:45:12 +0200
Subject: [PATCH 2/2] configure networking explicitly on each node

Due to a bug in kubernetes-nmstate/nmstate/NM, we get a random MAC
address on our bridge after reboot. Because of that we get a different
IP address and lose the host.

With this patch, we explicitly request MAC of the NIC on the bridge.

Signed-off-by: Petr Horacek <phoracek@redhat.com>
---
 CNV/configure-network.sh | 34 +++++++++++++++++++++-------------
 README.md                | 10 ++++++----
 2 files changed, 27 insertions(+), 17 deletions(-)

diff --git a/CNV/configure-network.sh b/CNV/configure-network.sh
index 2181475..ce84150 100755
--- a/CNV/configure-network.sh
+++ b/CNV/configure-network.sh
@@ -7,30 +7,38 @@ BRIDGE_NAME=brext
 
 export KUBECONFIG=${KUBECONFIG:-../OpenShift/ocp/auth/kubeconfig}
 
+nodes=$(oc get nodes -o jsonpath='{range .items[*]}{.metadata.name} {end}')
+
 echo "Configuring networks on nodes"
+for node in $nodes; do
+    echo "Detecting the default interface"
+    while ! default_iface=$(oc get nodenetworkstate ${node} -o jsonpath="{.status.currentState.routes.running[?(@.destination==\"${MACHINE_CIDR}\")].next-hop-interface}" | cut -d " " -f 1); do
+        sleep 10
+    done
 
-echo "Detecting the default interface"
-while ! default_iface=$(oc get nodenetworkstate ${node} -o jsonpath="{.items[0].status.currentState.routes.running[?(@.destination==\"${MACHINE_CIDR}\")].next-hop-interface}" | cut -d " " -f 1); do
-    sleep 10
-done
+    if [ "${default_iface}" == "${BRIDGE_NAME}" ]; then
+        echo "Bridge ${BRIDGE_NAME} seems to be already configured as the default interface on node ${node}, skipping the rest of network setup"
+        continue
+    fi
 
-if [ "${default_iface}" == "${BRIDGE_NAME}" ]; then
-    echo "Bridge ${BRIDGE_NAME} seems to be already configured as the default interface, skipping the rest of network setup"
-    exit 0
-fi
+    echo "Detecting MAC address of the default interface"
+    default_iface_mac=$(oc get nodenetworkstate ${node} -o jsonpath="{.status.currentState.interfaces[?(@.name==\"${default_iface}\")].mac-address}")
 
-echo "Applying node network configuration policy"
-cat <<EOF | oc apply -f -
+    echo "Applying node network configuration policy"
+    cat <<EOF | oc apply -f -
 apiVersion: nmstate.io/v1alpha1
 kind: NodeNetworkConfigurationPolicy
 metadata:
-  name: kni-policy
+  name: kni-${node}
 spec:
+  nodeSelector:
+    kubernetes.io/hostname: ${node}
   desiredState:
     interfaces:
     - name: ${BRIDGE_NAME}
       type: linux-bridge
       state: up
+      mac-address: ${default_iface_mac}
       ipv4:
         dhcp: true
         enabled: true
@@ -44,10 +52,10 @@ spec:
         port:
         - name: ${default_iface}
 EOF
+done
 
 echo "Waiting until the configuration is done, it may take up to 5 minutes until keepalived gets reconfigured"
-nodes=$(oc get nodes -o jsonpath='{range .items[*]}{.metadata.name} {end}')
 for node in $nodes; do
     until [ "$(oc get nodenetworkstate ${node} -o jsonpath="{.status.currentState.routes.running[?(@.destination==\"${MACHINE_CIDR}\")].next-hop-interface}")" == "${BRIDGE_NAME}" ]; do sleep 10; done
+    oc wait node ${node} --for condition=Ready --timeout=10m
 done
-oc wait node --all --for condition=Ready --timeout=10m
diff --git a/README.md b/README.md
index 59bff42..20629aa 100644
--- a/README.md
+++ b/README.md
@@ -83,10 +83,12 @@ The deployment process will use scripts to perform the following on the configur
    (#5)](https://github.com/openshift-kni/install-scripts/issues/5)
    for image storage.
 1. [Deploy
-   CNV](https://github.com/openshift-kni/install-scripts/blob/master/CNV/deploy-cnv.sh). [Configure
-   a bridge on the `External` interface on OpenShift nodes
-   (#18)](https://github.com/openshift-kni/install-scripts/issues/18)
-   to allow VMs access this network.
+   CNV](https://github.com/openshift-kni/install-scripts/blob/master/CNV/deploy-cnv.sh).
+1. [Configure host
+   networking](https://github.com/openshift-kni/install-scripts/blob/master/CNV/configure-network.sh)
+   by running `cd CNV && ./configure-network.sh`. That will create a linux
+   bridge on top of the default interface. Please note that when a new node is
+   added, the configuration script has to be retriggered.
 1. Temporarily install Ripsaw, carry out some performance tests, and
    capture the results.