USHIFT-3772: Dual stack RF e2e tests

scripts/devenv-builder/manage-vm.sh

@@ -22,7 +22,7 @@ SSH_PASSWORD_OPTS="-o PubkeyAuthentication=no -o PreferredAuthentications=passwo
 # Show the IP address of the VM
 function get_ip {
     sudo virsh domifaddr "$1" \
-        | grep vnet \
+        | grep ipv \
         | awk '{print $4}' \
         | cut -f1 -d/
 }

test/assets/dual-stack-network.xml

@@ -0,0 +1,18 @@
+<network>
+  <name>${VM_DUAL_STACK_NETWORK}</name>
+  <forward mode="nat">
+    <nat ipv6='yes'>
+      <port start='1024' end='65535'/>
+    </nat>
+  </forward>
+  <ip address='192.168.113.1' netmask='255.255.255.0' localPtr='yes'>
+    <dhcp>
+      <range start='192.168.113.100' end='192.168.113.254'/>
+    </dhcp>
+  </ip>
+  <ip family="ipv6" address="2001:db9:ca7:ff::1" prefix="96">
+    <dhcp>
+      <range start="2001:db9:ca7:ff::1000" end="2001:db9:ca7:ff::2000" />
+    </dhcp>
+  </ip>
+</network>

test/assets/hello-microshift-service-ipv6.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello-microshift
+  labels:
+    app: hello-microshift
+spec:
+  ipFamilies:
+  - IPv6
+  selector:
+    app: hello-microshift
+  ports:
+  - port: 8080
+    targetPort: 8080
+    protocol: TCP
+  type: ClusterIP

test/bin/common.sh

@@ -40,6 +40,10 @@ export VM_MULTUS_NETWORK="multus"
 # shellcheck disable=SC2034  # used elsewhere
 export VM_IPV6_NETWORK="ipv6"
 
+# Libvirt network for dual stack tests
+# shellcheck disable=SC2034  # used elsewhere
+export VM_DUAL_STACK_NETWORK="dual-stack"
+
 # Location of RPMs built from source
 # shellcheck disable=SC2034  # used elsewhere
 RPM_SOURCE="${OUTPUTDIR}/rpmbuild"

test/bin/manage_hypervisor_config.sh

@@ -28,17 +28,17 @@ firewall_settings() {
 
     sudo firewall-cmd --permanent --zone=libvirt "--${action}-service=mdns"
 
-    for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}"; do
+    for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}"; do
         if ! sudo virsh net-info "${netname}" &>/dev/null ; then
             continue
         fi
 
         local vm_bridge
-        local vm_bridge_cidr
         vm_bridge=$(sudo virsh net-dumpxml "${netname}" | yq -p xml '.network.bridge.+@name')
-        vm_bridge_cidr=$(ip addr show "${vm_bridge}" | grep "scope global" | awk '{print $2}')
 
-        sudo firewall-cmd --permanent --zone=trusted "--${action}-source"="${vm_bridge_cidr}"
+        for ip in $(ip addr show "${vm_bridge}" | grep "scope global" | awk '{print $2}'); do
+            sudo firewall-cmd --permanent --zone=trusted "--${action}-source"="${ip}"
+        done
         sudo firewall-cmd --permanent --zone=public  "--${action}-port"="${WEB_SERVER_PORT}/tcp"
         sudo firewall-cmd --reload
     done
@@ -97,6 +97,23 @@ action_create() {
         sudo ip link add name "${bridge_name}p0" up master "${bridge_name}" type dummy
     fi
 
+    if ! sudo sudo virsh net-info "${VM_DUAL_STACK_NETWORK}" &>/dev/null ; then
+        local -r dual_stack_netconfig_tmpl="${SCRIPTDIR}/../assets/dual-stack-network.xml"
+        local -r dual_stack_netconfig_file="${IMAGEDIR}/infra/dual-stack-network.xml"
+
+        mkdir -p "$(dirname "${dual_stack_netconfig_file}")"
+        envsubst <"${dual_stack_netconfig_tmpl}" >"${dual_stack_netconfig_file}"
+
+        sudo virsh net-define    "${dual_stack_netconfig_file}"
+        sudo virsh net-start     "${VM_DUAL_STACK_NETWORK}"
+        sudo virsh net-autostart "${VM_DUAL_STACK_NETWORK}"
+
+        # Add a dummy port so the bridge is not DOWN and the routing works without
+        # falling back through the default route
+        bridge_name=$(sudo virsh net-dumpxml ${VM_DUAL_STACK_NETWORK} | yq -p xml '.network.bridge.+@name')
+        sudo ip link add name "${bridge_name}p0" up master "${bridge_name}" type dummy
+    fi
+
     # Firewall
     firewall_settings "add"
 }
@@ -118,6 +135,13 @@ action_cleanup() {
         sudo virsh net-undefine "${VM_IPV6_NETWORK}"
     fi
 
+    if sudo virsh net-info "${VM_DUAL_STACK_NETWORK}" &>/dev/null ; then
+        bridge_name=$(sudo virsh net-dumpxml ${VM_DUAL_STACK_NETWORK} | yq -p xml '.network.bridge.+@name')
+        sudo ip link del name "${bridge_name}p0"
+        sudo virsh net-destroy "${VM_DUAL_STACK_NETWORK}"
+        sudo virsh net-undefine "${VM_DUAL_STACK_NETWORK}"
+    fi
+
     # Storage pool
     for pool_name in $(sudo virsh pool-list --name | awk '/vm-storage/ {print $1}') ; do       
         if sudo virsh pool-info "${pool_name}" &>/dev/null; then

test/resources/libipv6.py

@@ -26,6 +26,10 @@ def must_be_ipv6(addr: str) -> None:
     BuiltIn().should_be_true(is_ipv6(addr))
 
 
+def must_not_be_ipv6(addr: str) -> None:
+    BuiltIn().should_not_be_true(is_ipv6(addr))
+
+
 def add_brackets_if_ipv6(ip: str) -> str:
     """
     Add square brackets to the given IP if its ipv6 for later use in other tools (e.g. curl)

test/scenarios/el94-src@dual-stack.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+scenario_create_vms() {
+    prepare_kickstart host1 kickstart.ks.template rhel-9.4-microshift-source
+    launch_vm host1 rhel-9.4 "${VM_DUAL_STACK_NETWORK}"
+}
+
+scenario_remove_vms() {
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    local -r vmname=$(full_vm_name host1)
+    local -r vm_ip1=$("${ROOTDIR}/scripts/devenv-builder/manage-vm.sh" ip -n "${vmname}" | head -1)
+    local -r vm_ip2=$("${ROOTDIR}/scripts/devenv-builder/manage-vm.sh" ip -n "${vmname}" | tail -1)
+
+    run_tests host1 \
+        --variable "USHIFT_HOST_IP1:${vm_ip1}" \
+        --variable "USHIFT_HOST_IP2:${vm_ip2}" \
+        suites/ipv6/dualstack.robot
+}

test/suites/ipv6/dualstack.robot

@@ -0,0 +1,154 @@
+*** Settings ***
+Documentation       Tests related to MicroShift running in a dual stack ipv4+6 host
+
+Resource            ../../resources/common.resource
+Resource            ../../resources/oc.resource
+Resource            ../../resources/ostree-health.resource
+Resource            ../../resources/microshift-network.resource
+Resource            ../../resources/microshift-config.resource
+Library             ../../resources/libipv6.py
+
+Suite Setup         Setup
+Suite Teardown      Teardown
+
+Test Tags           ipv6    network
+
+
+*** Variables ***
+${USHIFT_HOST_IP1}      ${EMPTY}
+${USHIFT_HOST_IP2}      ${EMPTY}
+${HOSTNAME}             hello-microshift.dualstack.cluster.local
+
+
+*** Test Cases ***
+Verify New Pod Works With IPv6
+    [Documentation]    Verify IPv6 services are routable.
+    [Setup]    Run Keywords
+    ...    Save Default MicroShift Config
+    ...    Migrate To Dual Stack
+    ...    Create Hello MicroShift Pod
+    ...    Expose Hello MicroShift Service Via Route IPv6
+    ...    Restart Router
+
+    ${pod_ip}=    Oc Get JsonPath    pod    ${NAMESPACE}    hello-microshift    .status.podIPs[0].ip
+    Must Not Be Ipv6    ${pod_ip}
+    ${pod_ip}=    Oc Get JsonPath    pod    ${NAMESPACE}    hello-microshift    .status.podIPs[1].ip
+    Must Be Ipv6    ${pod_ip}
+    ${service_ip}=    Oc Get JsonPath    svc    ${NAMESPACE}    hello-microshift    .spec.clusterIP
+    Must Be Ipv6    ${service_ip}
+
+    Wait Until Keyword Succeeds    10x    6s
+    ...    Access Hello Microshift Success    ushift_ip=${USHIFT_HOST_IP1}
+    ...        ushift_port=${HTTP_PORT}
+    ...        hostname=${HOSTNAME}
+    Wait Until Keyword Succeeds    10x    6s
+    ...    Access Hello Microshift Success    ushift_ip=${USHIFT_HOST_IP2}
+    ...        ushift_port=${HTTP_PORT}
+    ...        hostname=${HOSTNAME}
+
+    [Teardown]    Run Keywords
+    ...    Delete Hello MicroShift Route
+    ...    Delete Hello MicroShift Pod And Service
+    ...    Wait For Service Deletion With Timeout
+    ...    Restore Default MicroShift Config
+    ...    Restart MicroShift
+
+Verify New Pod Works With IPv4
+    [Documentation]    Verify IPv4 services are routable.
+    [Setup]    Run Keywords
+    ...    Save Default MicroShift Config
+    ...    Migrate To Dual Stack
+    ...    Create Hello MicroShift Pod
+    ...    Expose Hello MicroShift Service Via Route IPv4
+    ...    Restart Router
+
+    ${pod_ip}=    Oc Get JsonPath    pod    ${NAMESPACE}    hello-microshift    .status.podIPs[0].ip
+    Must Not Be Ipv6    ${pod_ip}
+    ${pod_ip}=    Oc Get JsonPath    pod    ${NAMESPACE}    hello-microshift    .status.podIPs[1].ip
+    Must Be Ipv6    ${pod_ip}
+    ${service_ip}=    Oc Get JsonPath    svc    ${NAMESPACE}    hello-microshift    .spec.clusterIP
+    Must Not Be Ipv6    ${service_ip}
+
+    Wait Until Keyword Succeeds    10x    6s
+    ...    Access Hello Microshift Success    ushift_ip=${USHIFT_HOST_IP1}
+    ...        ushift_port=${HTTP_PORT}
+    ...        hostname=${HOSTNAME}
+    Wait Until Keyword Succeeds    10x    6s
+    ...    Access Hello Microshift Success    ushift_ip=${USHIFT_HOST_IP2}
+    ...        ushift_port=${HTTP_PORT}
+    ...        hostname=${HOSTNAME}
+
+    [Teardown]    Run Keywords
+    ...    Delete Hello MicroShift Route
+    ...    Delete Hello MicroShift Pod And Service
+    ...    Wait For Service Deletion With Timeout
+    ...    Restore Default MicroShift Config
+    ...    Restart MicroShift
+
+
+*** Keywords ***
+Setup
+    [Documentation]    Test suite setup
+    Initialize Global Variables
+    Login MicroShift Host
+    Setup Suite With Namespace
+    Wait Until Greenboot Health Check Exited
+
+Teardown
+    [Documentation]    Test suite teardown
+    Teardown Suite With Namespace
+    Logout MicroShift Host
+
+Initialize Global Variables
+    [Documentation]    Initializes global variables.
+    Log    IP1: ${USHIFT_HOST_IP1} IPv6: ${USHIFT_HOST_IP2}
+    Should Not Be Empty    ${USHIFT_HOST_IP1}    USHIFT_HOST_IP1 variable is required
+    Should Not Be Empty    ${USHIFT_HOST_IP2}    USHIFT_HOST_IP2 variable is required
+
+Migrate To Dual Stack
+    [Documentation]    Configure MicroShift to enable dual stack network
+
+    ${dual_stack}=    CATENATE    SEPARATOR=\n
+    ...    ---
+    ...    network:
+    ...    \ \ clusterNetwork: [10.42.0.0/16, fd01::/48]
+    ...    \ \ serviceNetwork: [10.43.0.0/16, fd02::/112]
+    ${replaced}=    Replace MicroShift Config    ${dual_stack}
+    Upload MicroShift Config    ${replaced}
+    Restart MicroShift
+
+Delete Hello MicroShift Route
+    [Documentation]    Delete route for cleanup.
+    Oc Delete    route/hello-microshift -n ${NAMESPACE}
+
+Wait For Service Deletion With Timeout
+    [Documentation]    Polls for service and endpoint by "app=hello-microshift" label. Fails if timeout
+    ...    expires. This check is unique to this test suite because each test here reuses the same namespace. Since
+    ...    the tests reuse the service name, a small race window exists between the teardown of one test and the setup
+    ...    of the next. This produces flakey failures when the service or endpoint names collide.
+    Wait Until Keyword Succeeds    30s    1s
+    ...    Network APIs With Test Label Are Gone
+
+Expose Hello MicroShift Service Via Route IPv4
+    [Documentation]    Expose the "hello microshift" application through the Route
+    Run With Kubeconfig    oc apply -n ${NAMESPACE} -f assets/hello-microshift-service.yaml
+    Oc Expose    svc hello-microshift --hostname ${HOSTNAME} -n ${NAMESPACE}
+
+Expose Hello MicroShift Service Via Route IPv6
+    [Documentation]    Expose the "hello microshift" application through the Route
+    Run With Kubeconfig    oc apply -n ${NAMESPACE} -f assets/hello-microshift-service-ipv6.yaml
+    Oc Expose    svc hello-microshift --hostname ${HOSTNAME} -n ${NAMESPACE}
+
+Network APIs With Test Label Are Gone
+    [Documentation]    Check for service and endpoint by "app=hello-microshift" label. Succeeds if response matches
+    ...    "No resources found in <namespace> namespace." Fail if not.
+    ${match_string}=    Catenate    No resources found in    ${NAMESPACE}    namespace.
+    ${match_string}=    Remove String    ${match_string}    "
+    ${response}=    Run With Kubeconfig    oc get svc,ep -l app\=hello-microshift -n ${NAMESPACE}
+    Should Be Equal As Strings    ${match_string}    ${response}    strip_spaces=True
+
+Restart Router
+    [Documentation]    Restart the router and wait for readiness again. The router is sensitive to apiserver
+    ...    downtime and might need a restart (after the apiserver is ready) to resync all the routes.
+    Run With Kubeconfig    oc rollout restart deployment router-default -n openshift-ingress
+    Named Deployment Should Be Available    router-default    openshift-ingress    5m