From 7fc9461c9e39babcfbccac52a27650383e043c26 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Fri, 20 Dec 2019 17:19:52 +0000 Subject: [PATCH 01/22] WIP ipv6 testing --- 06_create_cluster.sh | 12 ++++++++++-- ocp_install_env.sh | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/06_create_cluster.sh b/06_create_cluster.sh index b90f8df74..5b3ef496c 100755 --- a/06_create_cluster.sh +++ b/06_create_cluster.sh @@ -23,13 +23,21 @@ fi # NOTE: This is equivalent to the external API DNS record pointing the API to the API VIP if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then - API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then + API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + else + API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + fi INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))") echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift.conf echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf sudo systemctl reload NetworkManager else - API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then + API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + else + API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + fi INGRESS_VIP=$(dig +noall +answer "test.apps.${CLUSTER_DOMAIN}" | awk '{print $NF}') fi diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 517cfef5a..5baec69a8 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -53,6 +53,11 @@ function clone_installer() { if [[ ! -d $OPENSHIFT_INSTALL_PATH ]]; then sync_repo_and_patch go/src/github.com/openshift/installer https://github.com/openshift/installer.git fi + pushd $OPENSHIFT_INSTALL_PATH + git remote add kni https://github.com/openshift-kni/installer.git || true + git fetch -v kni + git checkout -b kni/4.3-ipv6-$$ kni/4.3-ipv6 + popd } function build_installer() { From c8a3dc6fabb40508342fc89ec0d3549c19890647 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Mon, 23 Dec 2019 16:22:02 +0000 Subject: [PATCH 02/22] Run registry with host networking So we can access it via ipv6 --- utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.sh b/utils.sh index 76f77de70..e6dcb96ce 100644 --- a/utils.sh +++ b/utils.sh @@ -304,7 +304,7 @@ EOF if [[ "$reg_state" != "running" || "$SSL_CERT_MD5_HASH" != "$MD5_HASH_RUNNING" ]]; then sudo podman rm registry -f || true - sudo podman run -d --name registry -p ${LOCAL_REGISTRY_PORT}:5000 \ + sudo podman run -d --name registry --net=host --privileged \ -v ${REGISTRY_DIR}/data:/var/lib/registry:z \ -v ${REGISTRY_DIR}/auth:/auth:z \ -e "REGISTRY_AUTH=htpasswd" \ From e2e0fac8fc481557fcab227ae1386a1ede8b6597 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Tue, 31 Dec 2019 10:55:43 +0000 Subject: [PATCH 03/22] Use ipcalc to validate IP address Instead of a regex which only accepts ipv4 addresses --- utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.sh b/utils.sh index e6dcb96ce..2cb6656d5 100644 --- a/utils.sh +++ b/utils.sh @@ -237,7 +237,7 @@ function setup_local_registry() { pushd $REGISTRY_DIR/certs SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}" - if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then + if ipcalc -c $SSL_HOST_NAME; then SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}" else From 48c7d732d28295d7caa091ee6a066794354cec60 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Fri, 3 Jan 2020 18:25:59 +0000 Subject: [PATCH 04/22] Reference mirror registry by name, not IP Since the oc adm mirror command appears to reject target registries and pullspecs that contain ipv6 addresses[1] we can work around this by using a name instead [1] https://github.com/openshift/oc/issues/239 --- 01_install_requirements.sh | 1 + 02_configure_host.sh | 2 +- 04_setup_ironic.sh | 8 ++++---- 06_create_cluster.sh | 2 +- common.sh | 3 +-- config_example.sh | 2 +- ocp_install_env.sh | 1 + utils.sh | 8 ++++---- vm_setup_vars.yml | 3 +++ 9 files changed, 17 insertions(+), 13 deletions(-) diff --git a/01_install_requirements.sh b/01_install_requirements.sh index 5bc157d51..1adb9adc3 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -4,6 +4,7 @@ set -ex source logging.sh source common.sh source utils.sh +source ocp_install_env.sh if grep -q "Red Hat Enterprise Linux release 8" /etc/redhat-release 2>/dev/null ; then RHEL8="True" diff --git a/02_configure_host.sh b/02_configure_host.sh index 83f229b1d..3dd7a3104 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -176,7 +176,7 @@ if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then # create authfile for local registry sudo podman login --authfile ${REGISTRY_CREDS} \ -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \ - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT} + ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT} else # Create a blank authfile in order to have something valid when we read it in 04_setup_ironic.sh echo '{}' | sudo dd of=${REGISTRY_CREDS} diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index b78edc23c..75140c671 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -39,7 +39,7 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH cd $REPOPATH export $IMAGE_VAR=${IMAGE##*/}:latest - export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} + export $IMAGE_VAR=$LOCAL_REGISTRY_DNS_NAME:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} . cd - sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR} @@ -65,15 +65,15 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then --insecure=true \ -a ${COMBINED_AUTH_FILE} \ --from ${OPENSHIFT_RELEASE_IMAGE} \ - --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ - --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} + --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ + --to ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected, #you must extract the installation program from the mirrored content: if [ -z "$KNI_INSTALL_FROM_GIT" ]; then oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \ --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \ - "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" + "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/ fi diff --git a/06_create_cluster.sh b/06_create_cluster.sh index 5b3ef496c..1ef5fb9a8 100755 --- a/06_create_cluster.sh +++ b/06_create_cluster.sh @@ -29,7 +29,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') fi INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))") - echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift.conf + echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf sudo systemctl reload NetworkManager else diff --git a/common.sh b/common.sh index f9468396b..40ccf5ed3 100644 --- a/common.sh +++ b/common.sh @@ -42,7 +42,6 @@ export MIRROR_IMAGES=${MIRROR_IMAGES:-} WORKING_DIR=${WORKING_DIR:-"/opt/dev-scripts"} # variables for local registry configuration -export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"} export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"} export REGISTRY_USER=${REGISTRY_USER:-ocp-user} export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass} @@ -94,7 +93,7 @@ fi if [ -n "$MIRROR_IMAGES" ]; then # We're going to be using a locally modified release image - export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" + export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" fi # Set variables diff --git a/config_example.sh b/config_example.sh index 5fcf22b54..5cb844ab6 100644 --- a/config_example.sh +++ b/config_example.sh @@ -76,7 +76,7 @@ set -x # Name of branch in the above repo which contains the custom MAO changes #export MAO_BRANCH="mao-fix" -#export LOCAL_REGISTRY_ADDRESS="192.168.111.1" +#export LOCAL_REGISTRY_DNS_NAME="virthost.ostest.test.metalkube.org" #export LOCAL_REGISTRY_PORT="5000" # configure username for registry diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 5baec69a8..668240744 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -8,6 +8,7 @@ export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"} export DNS_VIP=${DNS_VIP:-"192.168.111.2"} +export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"} function extract_command() { local release_image diff --git a/utils.sh b/utils.sh index 2cb6656d5..50a8cc6c8 100644 --- a/utils.sh +++ b/utils.sh @@ -207,10 +207,10 @@ function image_mirror_config { cat << EOF imageContentSources: - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${RELEASE} - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${TAGGED} additionalTrustBundle: | ${INDENTED_CERT} @@ -235,13 +235,13 @@ function setup_local_registry() { sudo chown -R $USER:$USER ${REGISTRY_DIR} pushd $REGISTRY_DIR/certs - SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}" + SSL_HOST_NAME="${LOCAL_REGISTRY_DNS_NAME}" if ipcalc -c $SSL_HOST_NAME; then SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}" else - SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}" + SSL_EXT_8="subjectAltName = DNS:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}" fi diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml index 7f0062ada..b95f96505 100644 --- a/vm_setup_vars.yml +++ b/vm_setup_vars.yml @@ -31,3 +31,6 @@ dns_extrahosts: - ip: "{{ baremetal_network_cidr | nthhost(2) }}" hostnames: - "ns1" + - ip: "{{ baremetal_network_cidr | nthhost(1) }}" + hostnames: + - "virthost" From 30ba148cbf962df76fde6df4376a0f2784e0f5de Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Thu, 9 Jan 2020 14:24:05 +0100 Subject: [PATCH 05/22] Use latest tag instead of release one This is needed to work with ipv6 image, as it doesn't have latest tag and bootstrap tries to use it. --- 04_setup_ironic.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index 75140c671..bb876e6c2 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -65,7 +65,7 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then --insecure=true \ -a ${COMBINED_AUTH_FILE} \ --from ${OPENSHIFT_RELEASE_IMAGE} \ - --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ + --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest \ --to ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected, @@ -73,7 +73,7 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then if [ -z "$KNI_INSTALL_FROM_GIT" ]; then oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \ --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \ - "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" + "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/ fi From 8aadd748f883847aee60c5bdb6987bb62a4453b3 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Thu, 9 Jan 2020 14:12:20 +0000 Subject: [PATCH 06/22] Move vars fixup --- common.sh | 10 ++++++++++ ocp_install_env.sh | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/common.sh b/common.sh index 40ccf5ed3..ea4d28dda 100644 --- a/common.sh +++ b/common.sh @@ -36,6 +36,16 @@ if [ -z "${CONFIG:-}" ]; then fi source $CONFIG +export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org} +export CLUSTER_NAME=${CLUSTER_NAME:-ostest} +export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" +export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" +export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} +export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} +export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"} +export DNS_VIP=${DNS_VIP:-"192.168.111.2"} +export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"} + # mirror images for installation in restricted network export MIRROR_IMAGES=${MIRROR_IMAGES:-} diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 668240744..7557284a9 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -1,15 +1,5 @@ eval "$(go env)" -export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org} -export CLUSTER_NAME=${CLUSTER_NAME:-ostest} -export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" -export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" -export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} -export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} -export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"} -export DNS_VIP=${DNS_VIP:-"192.168.111.2"} -export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"} - function extract_command() { local release_image local cmd From 30fec8eb9921fbd0dfe645a843f9ee435e710ce0 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Mon, 13 Jan 2020 18:28:44 +0000 Subject: [PATCH 07/22] Pass DHCP options to force DNS to use the unique local address According to the manpage [fd00::] gets expanded to the non link-local address --- vm_setup_vars.yml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml index b95f96505..7d3137549 100644 --- a/vm_setup_vars.yml +++ b/vm_setup_vars.yml @@ -22,8 +22,41 @@ flavors: vcpu: 4 extradisks: false -# For OpenShift we create some additional DNS records for the API/DNS VIPs + baremetal_network_cidr: "{{ lookup('env', 'EXTERNAL_SUBNET') | default('192.168.111.0/24', true) }}" + +# Describe our virtual networks. These networks will be attached to +# the vm nodes in the order in which they are defined with the following caveats: +# * The first bridge network defined will be used for pxe booting +manage_baremetal: 'y' +networks: + - name: provisioning + bridge: provisioning + forward_mode: bridge + - name: baremetal + bridge: baremetal + forward_mode: "{% if manage_baremetal == 'y' %}nat{% else %}bridge{% endif %}" + address: "{{ baremetal_network_cidr|nthhost(1) }}" + netmask: "{{ baremetal_network_cidr|ipaddr('netmask') }}" + prefix: "{{ baremetal_network_cidr|ipaddr('prefix') }}" + dhcp_range: + - "{{ baremetal_network_cidr|nthhost(20) }}" + - "{{ baremetal_network_cidr|nthhost(60) }}" + nat_port_range: + - 1024 + - 65535 + domain: "{{ cluster_domain }}" + dns: + # http://www.thekelleys.org.uk/dnsmasq/docs/dnsmasq-man.html + # [fd00::] is replaced with the Unique Local Address + options: 'dhcp-option=option6:dns-server,[fd00::]' + hosts: "{{dns_extrahosts | default([])}}" + forwarders: + - domain: "apps.{{ cluster_domain }}" + addr: "{% if baremetal_network_cidr|ipv6 != False %}::1{% else %}127.0.0.1{% endif %}" + + +# For OpenShift we create some additional DNS records for the API/DNS VIPs dns_extrahosts: - ip: "{{ baremetal_network_cidr | nthhost(5) }}" hostnames: From 9df40b42402a46d14078071a5d1eba470d9c9fb9 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Fri, 17 Jan 2020 21:27:17 -0500 Subject: [PATCH 08/22] Configure IPv6 subnets in install-config.yaml --- ocp_install_env.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 7557284a9..221c9bb8e 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -87,6 +87,24 @@ baseDomain: ${BASE_DOMAIN} networking: networkType: ${NETWORK_TYPE} machineCIDR: ${EXTERNAL_SUBNET} +EOF + + # IPv6 network config + if [[ "${EXTERNAL_SUBNET}" == *":"* ]]; then + if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then + echo "NETWORK_TYPE must be OVNKubernetes when using IPv6" + exit 1 + fi + cat >> "${outdir}/install-config.yaml" << EOF + clusterNetwork: + - cidr: fd01::/48 + hostPrefix: 64 + serviceNetwork: + - fd02::/112 +EOF + fi + + cat >> "${outdir}/install-config.yaml" << EOF metadata: name: ${CLUSTER_NAME} compute: From 5af1bdc07c30132e1b008fd98108fcd5de42c7d6 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Wed, 15 Jan 2020 14:37:21 -0500 Subject: [PATCH 09/22] Run ifup on provisioning bridge after bouncing interface On RHEL 8, when ifdown is run on a bridge's only (or last up) interface, then the bridge is deleted. However, when ifup is run on the bridge's interface, it is not correspondingly run on the bridge itself. See: https://github.com/fedora-sysv/initscripts/blob/rhel8-branch/network-scripts/ifdown-eth#L144 Since the provisioning interface is bounced with ifup/ifdown after bringing the bridge up, then the bridge itself ends up not existing. This patch adds an additional call to ifup the provisioning bridge after bouncing the interface. Signed-off-by: James Slagle --- 02_configure_host.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/02_configure_host.sh b/02_configure_host.sh index 3dd7a3104..7451bbc04 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -86,6 +86,9 @@ if [ "$MANAGE_PRO_BRIDGE" == "y" ]; then echo -e "DEVICE=$PRO_IF\nTYPE=Ethernet\nONBOOT=yes\nNM_CONTROLLED=no\nBRIDGE=provisioning" | sudo dd of=/etc/sysconfig/network-scripts/ifcfg-$PRO_IF sudo ifdown $PRO_IF || true sudo ifup $PRO_IF + # Need to ifup the provisioning bridge again because ifdown $PRO_IF + # will bring down the bridge as well. + sudo ifup provisioning fi fi From 9bda645eeb7ec41fe110d2fb66dc56ae8a049d74 Mon Sep 17 00:00:00 2001 From: Derek Higgins Date: Tue, 14 Jan 2020 10:12:43 -0500 Subject: [PATCH 10/22] Always build a local release image if mirroring Both MIRROR_IMAGES=true and *_LOCAL_IMAGE rely on this image being build so we should build it in both cases. We had only been building it if *_LOCAL_IMAGE was set. Fixes: #880 --- 04_setup_ironic.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index bb876e6c2..227d29093 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -23,11 +23,9 @@ export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX") COMBINED_AUTH_FILE=$(mktemp "combined-pullsecret--XXXXXXXXXX") jq -s '.[0] * .[1]' ${REGISTRY_AUTH_FILE} ${REGISTRY_CREDS} | tee ${COMBINED_AUTH_FILE} -_local_images= DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX") echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do - _local_images=1 IMAGE=${!IMAGE_VAR} sudo -E podman pull --authfile $COMBINED_AUTH_FILE $OPENSHIFT_RELEASE_IMAGE @@ -79,9 +77,8 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then fi rm -rf "${EXTRACT_DIR}" -fi -if [ "${_local_images}" == "1" ]; then + # Build a local release image, if no *_LOCAL_IMAGE env variables are set then this is just a copy of the release image sudo podman image build --authfile $COMBINED_AUTH_FILE -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE fi From 0601a03083a5fb324f5725562042cd71bb4e5fe1 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Fri, 17 Jan 2020 13:19:27 +0000 Subject: [PATCH 11/22] WIP Remove dns options to force fd00: This should not be needed since we now prepend the DNS VIP via the MCO, but I'm not clear yet if we should leave this anyway as it stops NetworkManager adding the link-local IP to resolv.conf? --- vm_setup_vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml index 7d3137549..46923dd80 100644 --- a/vm_setup_vars.yml +++ b/vm_setup_vars.yml @@ -49,7 +49,7 @@ networks: dns: # http://www.thekelleys.org.uk/dnsmasq/docs/dnsmasq-man.html # [fd00::] is replaced with the Unique Local Address - options: 'dhcp-option=option6:dns-server,[fd00::]' + #options: 'dhcp-option=option6:dns-server,[fd00::]' hosts: "{{dns_extrahosts | default([])}}" forwarders: - domain: "apps.{{ cluster_domain }}" From 2a744aadf839134f27e21468f62b90d6af3df5ca Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Fri, 17 Jan 2020 13:21:47 +0000 Subject: [PATCH 12/22] ocp_cleanup: Remove any stale .ign volumes These can get left as the --remove-all-storage doesn't remove them (they're passed via -fw-cfg not strictly owned by the domain). --- ocp_cleanup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ocp_cleanup.sh b/ocp_cleanup.sh index 7fa10abb3..16d1bfa53 100755 --- a/ocp_cleanup.sh +++ b/ocp_cleanup.sh @@ -35,6 +35,10 @@ for vm in $(sudo virsh list --all --name | grep "^${CLUSTER_NAME}.*bootstrap"); sudo virsh destroy $vm sudo virsh undefine $vm --remove-all-storage done +# The .ign volume isn't deleted via --remove-all-storage +for v in $(sudo virsh vol-list --pool default | grep "^${CLUSTER_NAME}.*bootstrap" | awk '{print $1}'); do + sudo virsh vol-delete $v --pool default +done if [ -d assets/generated ]; then rm -rf assets/generated From b70a301e49a4001ec52b1d65e4d1508a4644ba97 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Mon, 20 Jan 2020 11:50:07 +0000 Subject: [PATCH 13/22] Make cluster/service subnets configurable Default to the values from the installer for ipv4, but we can override like this for ipv6: export CLUSTER_SUBNET="fd01::/48" export CLUSTER_HOST_PREFIX="64" export SERVICE_SUBNET="fd02::/112" --- common.sh | 3 +++ ocp_install_env.sh | 27 +++++++++++---------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/common.sh b/common.sh index ea4d28dda..a31c2e9bc 100644 --- a/common.sh +++ b/common.sh @@ -42,6 +42,9 @@ export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} +export CLUSTER_SUBNET=${CLUSTER_SUBNET:-"10.128.0.0/14"} +export CLUSTER_HOST_PREFIX=${CLUSTER_HOST_PREFIX:-"23"} +export SERVICE_SUBNET=${SERVICE_SUBNET:-"172.30.0.0/16"} export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"} export DNS_VIP=${DNS_VIP:-"192.168.111.2"} export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"} diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 221c9bb8e..12f4c7926 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -81,30 +81,25 @@ function generate_ocp_install_config() { fi mkdir -p "${outdir}" + + # IPv6 network config validation + if [[ "${EXTERNAL_SUBNET}" =~ .*:.* ]]; then + if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then + echo "NETWORK_TYPE must be OVNKubernetes when using IPv6" + exit 1 + fi + fi cat > "${outdir}/install-config.yaml" << EOF apiVersion: v1 baseDomain: ${BASE_DOMAIN} networking: networkType: ${NETWORK_TYPE} machineCIDR: ${EXTERNAL_SUBNET} -EOF - - # IPv6 network config - if [[ "${EXTERNAL_SUBNET}" == *":"* ]]; then - if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then - echo "NETWORK_TYPE must be OVNKubernetes when using IPv6" - exit 1 - fi - cat >> "${outdir}/install-config.yaml" << EOF clusterNetwork: - - cidr: fd01::/48 - hostPrefix: 64 + - cidr: ${CLUSTER_SUBNET} + hostPrefix: ${CLUSTER_HOST_PREFIX} serviceNetwork: - - fd02::/112 -EOF - fi - - cat >> "${outdir}/install-config.yaml" << EOF + - ${SERVICE_SUBNET} metadata: name: ${CLUSTER_NAME} compute: From 519878c516692894265a59cd1273c8d7ad80fdaa Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 20 Jan 2020 10:46:28 -0500 Subject: [PATCH 14/22] Add v6workarounds.sh This script includes the current set of workarounds needed to get a working etcd cluster. --- v6workarounds.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 v6workarounds.sh diff --git a/v6workarounds.sh b/v6workarounds.sh new file mode 100755 index 000000000..d458ef551 --- /dev/null +++ b/v6workarounds.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ $# != 2 ] ; then + echo "Usage: ./v6workarounds.sh " + exit 1 +fi + +# The BOOTSTRAP_IP isn't always the same, so we have to take it as an argument +BOOTSTRAP_IP="$1" +DNS_VIP=$2 + +MASTER_IPS="fd2e:6f44:5dd8:c956::14 fd2e:6f44:5dd8:c956::15 fd2e:6f44:5dd8:c956::16" + +fssh() { + ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $@ +} + +for MASTER in ${MASTER_IPS} ; do + fssh core@${MASTER} sudo systemctl restart NetworkManager + sleep 5 + fssh core@${MASTER} hostname + MDNS_POD=$(fssh core@${MASTER} sudo crictl pods | grep mdns | awk '{print $1}') + fssh core@${MASTER} sudo crictl stopp ${MDNS_POD} + fssh core@${MASTER} sudo crictl rmp ${MDNS_POD} + sleep 5 + fssh core@${MASTER} cat /etc/mdns/config.hcl | grep host +done + +fssh core@${BOOTSTRAP_IP} sudo sed -i \"1s/^/nameserver ${DNS_VIP}\\n/\" /etc/resolv.conf From 25c13f0148e2d90fcae077d10d986f018aa063bb Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 20 Jan 2020 10:52:40 -0500 Subject: [PATCH 15/22] v6workarounds.sh: Restart kubelet, add comments --- v6workarounds.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/v6workarounds.sh b/v6workarounds.sh index d458ef551..e487c40e6 100755 --- a/v6workarounds.sh +++ b/v6workarounds.sh @@ -16,14 +16,21 @@ fssh() { } for MASTER in ${MASTER_IPS} ; do + # Restart NetworkManager to get the correct hostname fssh core@${MASTER} sudo systemctl restart NetworkManager sleep 5 fssh core@${MASTER} hostname + + # Kill the mdns-publisher pod so that it runs its config init container again, + # and configures mdns-publisher with the correct hostname MDNS_POD=$(fssh core@${MASTER} sudo crictl pods | grep mdns | awk '{print $1}') fssh core@${MASTER} sudo crictl stopp ${MDNS_POD} fssh core@${MASTER} sudo crictl rmp ${MDNS_POD} sleep 5 fssh core@${MASTER} cat /etc/mdns/config.hcl | grep host + + # kubelet needs to be restarted to pick up the correct hotsname + fssh core@${MASTER} sudo systemctl restart kubelet done fssh core@${BOOTSTRAP_IP} sudo sed -i \"1s/^/nameserver ${DNS_VIP}\\n/\" /etc/resolv.conf From 6fe90bd5cf3c4ba438c309dc1b41610513e21b7c Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 20 Jan 2020 11:20:16 -0500 Subject: [PATCH 16/22] v6workarounds.sh: Make haproxy listen on IPv6 --- v6workarounds.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/v6workarounds.sh b/v6workarounds.sh index e487c40e6..b69d7cf81 100755 --- a/v6workarounds.sh +++ b/v6workarounds.sh @@ -31,6 +31,10 @@ for MASTER in ${MASTER_IPS} ; do # kubelet needs to be restarted to pick up the correct hotsname fssh core@${MASTER} sudo systemctl restart kubelet + + # haproxy not configured to listen on IPv6 + fssh core@${MASTER} sudo sed -i \"s/bind :7443/bind :::7443 v4v6/\" /etc/haproxy/haproxy.cfg + fssh core@${MASTER} sudo sed -i \"s/bind :50936/bind :::50936 v4v6/\" /etc/haproxy/haproxy.cfg done fssh core@${BOOTSTRAP_IP} sudo sed -i \"1s/^/nameserver ${DNS_VIP}\\n/\" /etc/resolv.conf From 0397402f872e12a52df6ccbc5f699cbde9d2cd56 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 20 Jan 2020 21:14:37 -0500 Subject: [PATCH 17/22] Add config_ipv6.sh Add a reference config file where we keep track of the settings being used by those testing the IPv6 test release images. --- config_ipv6.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 config_ipv6.sh diff --git a/config_ipv6.sh b/config_ipv6.sh new file mode 100644 index 000000000..8e04b49f4 --- /dev/null +++ b/config_ipv6.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Get a valid pull secret (json string) from +# You can get this secret from https://cloud.openshift.com/clusters/install#pull-secret +export PULL_SECRET=$(cat ~/pull-secret.json) + +export WORKING_DIR="$HOME" +# Local checkout with https://github.com/metal3-io/metal3-dev-env/pull/160 applied +export METAL3_DEV_ENV="$WORKING_DIR/metal3-dev-env" +export MIRROR_IMAGES=true +export OPENSHIFT_RELEASE_IMAGE="registry.svc.ci.openshift.org/ipv6/release:4.3.0-0.nightly-2020-01-16-123848-ipv6.5" +export EXTERNAL_SUBNET="fd2e:6f44:5dd8:c956::/120" +export DNS_VIP="fd2e:6f44:5dd8:c956:0:0:0:2" +export NETWORK_TYPE="OVNKubernetes" +export CLUSTER_SUBNET="fd01::/48" +export CLUSTER_HOST_PREFIX="64" +export SERVICE_SUBNET="fd02::/112" + +# Modify downloaded rhcos images to work around: +# https://bugzilla.redhat.com/show_bug.cgi?id=1787620 +# +# cd $WORKING_DIR/ironic/html/images +# gunzip rhcos-43.81.201912131630.0-qemu.x86_64.qcow2.gz +# virt-edit -a rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 -m /dev/sda1 -e "s/ip=any/ip=ens3:dhcp6/g" /grub2/grub.cfg +# sha256sum rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 +# gzip rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 +# sha256sum rhcos-43.81.201912131630.0-qemu.x86_64.qcow2.gz +# update .sha256sum file + +export MACHINE_OS_BOOTSTRAP_IMAGE_UNCOMPRESSED_SHA256="21e424f6ebfef68171d38088caeb1365d3b11e3d9f492304a3b7d6704c2b59fa" +export MACHINE_OS_BOOTSTRAP_IMAGE_SHA256="307ed6f8675fcc5b5f62f4711ebc55b54bdfb9175decff357241f3fb979d564c" +#export MACHINE_OS_IMAGE_UNCOMPRESSED_SHA256="d41814d65f300222dde0cf3c59a04c37b459cb3f4fb696da02832af212cdc8d3" +#export MACHINE_OS_IMAGE_SHA256="6c7016ec68d46c07937949644d6bbc74d4f55ebdb83872bce77f47ce2ee6d671" From ab50454d13ca26ed1bc96193336b0ba7264a88c1 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Wed, 22 Jan 2020 08:33:19 -0500 Subject: [PATCH 18/22] config_ipv6.sh: Don't set WORKING_DIR=$HOME This is a bad example. See issue #895. --- config_ipv6.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/config_ipv6.sh b/config_ipv6.sh index 8e04b49f4..f72a5b2bd 100644 --- a/config_ipv6.sh +++ b/config_ipv6.sh @@ -4,7 +4,6 @@ # You can get this secret from https://cloud.openshift.com/clusters/install#pull-secret export PULL_SECRET=$(cat ~/pull-secret.json) -export WORKING_DIR="$HOME" # Local checkout with https://github.com/metal3-io/metal3-dev-env/pull/160 applied export METAL3_DEV_ENV="$WORKING_DIR/metal3-dev-env" export MIRROR_IMAGES=true From afeca0cbc5c1bcfda1e7d7edd226045964d85077 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Wed, 22 Jan 2020 09:00:35 -0500 Subject: [PATCH 19/22] v6workarounds.sh: Drop hostname workarounds MCO has been updated as of 4.3.0-0.nightly-2020-01-16-123848-ipv6.6 to work around this issue without external workarounds. --- v6workarounds.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/v6workarounds.sh b/v6workarounds.sh index b69d7cf81..54dbfc999 100755 --- a/v6workarounds.sh +++ b/v6workarounds.sh @@ -16,22 +16,6 @@ fssh() { } for MASTER in ${MASTER_IPS} ; do - # Restart NetworkManager to get the correct hostname - fssh core@${MASTER} sudo systemctl restart NetworkManager - sleep 5 - fssh core@${MASTER} hostname - - # Kill the mdns-publisher pod so that it runs its config init container again, - # and configures mdns-publisher with the correct hostname - MDNS_POD=$(fssh core@${MASTER} sudo crictl pods | grep mdns | awk '{print $1}') - fssh core@${MASTER} sudo crictl stopp ${MDNS_POD} - fssh core@${MASTER} sudo crictl rmp ${MDNS_POD} - sleep 5 - fssh core@${MASTER} cat /etc/mdns/config.hcl | grep host - - # kubelet needs to be restarted to pick up the correct hotsname - fssh core@${MASTER} sudo systemctl restart kubelet - # haproxy not configured to listen on IPv6 fssh core@${MASTER} sudo sed -i \"s/bind :7443/bind :::7443 v4v6/\" /etc/haproxy/haproxy.cfg fssh core@${MASTER} sudo sed -i \"s/bind :50936/bind :::50936 v4v6/\" /etc/haproxy/haproxy.cfg From c00307c43bffcb4bda6b859b9ace63d6d9e40d7f Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Wed, 22 Jan 2020 13:48:20 -0500 Subject: [PATCH 20/22] Make dnsmasq listed on IPv6 The default configuration listens on IPv4 only. With this configuration, it seems to be listening on both IPv4 and IPv6 (localhost), so this should be a safe default in all cases for dev-scripts. --- 06_create_cluster.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/06_create_cluster.sh b/06_create_cluster.sh index 1ef5fb9a8..2d334084c 100755 --- a/06_create_cluster.sh +++ b/06_create_cluster.sh @@ -31,6 +31,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))") echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf + echo "listen-address=::1" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf sudo systemctl reload NetworkManager else if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then From f163a80f45b65276a0832053f39ecbab782f5970 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 23 Jan 2020 10:39:35 -0500 Subject: [PATCH 21/22] ipv6: Remove workaround for haproxy configs on masters This is handled automatically as of 4.3.0-0.nightly-2020-01-21-205041-ipv6.3 --- v6workarounds.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/v6workarounds.sh b/v6workarounds.sh index 54dbfc999..3fb476c2b 100755 --- a/v6workarounds.sh +++ b/v6workarounds.sh @@ -9,16 +9,8 @@ fi BOOTSTRAP_IP="$1" DNS_VIP=$2 -MASTER_IPS="fd2e:6f44:5dd8:c956::14 fd2e:6f44:5dd8:c956::15 fd2e:6f44:5dd8:c956::16" - fssh() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $@ } -for MASTER in ${MASTER_IPS} ; do - # haproxy not configured to listen on IPv6 - fssh core@${MASTER} sudo sed -i \"s/bind :7443/bind :::7443 v4v6/\" /etc/haproxy/haproxy.cfg - fssh core@${MASTER} sudo sed -i \"s/bind :50936/bind :::50936 v4v6/\" /etc/haproxy/haproxy.cfg -done - fssh core@${BOOTSTRAP_IP} sudo sed -i \"1s/^/nameserver ${DNS_VIP}\\n/\" /etc/resolv.conf From b98ad6749549c639a7980e6211c5df3324452b06 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Thu, 23 Jan 2020 20:45:15 -0500 Subject: [PATCH 22/22] Remove v6workarounds.sh This is no longer needed as of 4.3.0-0.nightly-2020-01-21-205041-ipv6.4 --- v6workarounds.sh | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100755 v6workarounds.sh diff --git a/v6workarounds.sh b/v6workarounds.sh deleted file mode 100755 index 3fb476c2b..000000000 --- a/v6workarounds.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -if [ $# != 2 ] ; then - echo "Usage: ./v6workarounds.sh " - exit 1 -fi - -# The BOOTSTRAP_IP isn't always the same, so we have to take it as an argument -BOOTSTRAP_IP="$1" -DNS_VIP=$2 - -fssh() { - ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $@ -} - -fssh core@${BOOTSTRAP_IP} sudo sed -i \"1s/^/nameserver ${DNS_VIP}\\n/\" /etc/resolv.conf