diff --git a/scripts/devenv-builder/manage-vm.sh b/scripts/devenv-builder/manage-vm.sh index 647fd04201..b4f77f9e39 100755 --- a/scripts/devenv-builder/manage-vm.sh +++ b/scripts/devenv-builder/manage-vm.sh @@ -22,7 +22,7 @@ SSH_PASSWORD_OPTS="-o PubkeyAuthentication=no -o PreferredAuthentications=passwo # Show the IP address of the VM function get_ip { sudo virsh domifaddr "$1" \ - | grep vnet \ + | grep ipv \ | awk '{print $4}' \ | cut -f1 -d/ } diff --git a/test/assets/dual-stack-network.xml b/test/assets/dual-stack-network.xml new file mode 100644 index 0000000000..de13b90f81 --- /dev/null +++ b/test/assets/dual-stack-network.xml @@ -0,0 +1,18 @@ + + ${VM_DUAL_STACK_NETWORK} + + + + + + + + + + + + + + + + diff --git a/test/assets/hello-microshift-service-ipv6.yaml b/test/assets/hello-microshift-service-ipv6.yaml new file mode 100644 index 0000000000..ac404eb2b2 --- /dev/null +++ b/test/assets/hello-microshift-service-ipv6.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: hello-microshift + labels: + app: hello-microshift +spec: + ipFamilies: + - IPv6 + selector: + app: hello-microshift + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + type: ClusterIP diff --git a/test/bin/common.sh b/test/bin/common.sh index c7936ebbee..4adc044c85 100644 --- a/test/bin/common.sh +++ b/test/bin/common.sh @@ -40,6 +40,10 @@ export VM_MULTUS_NETWORK="multus" # shellcheck disable=SC2034 # used elsewhere export VM_IPV6_NETWORK="ipv6" +# Libvirt network for dual stack tests +# shellcheck disable=SC2034 # used elsewhere +export VM_DUAL_STACK_NETWORK="dual-stack" + # Location of RPMs built from source # shellcheck disable=SC2034 # used elsewhere RPM_SOURCE="${OUTPUTDIR}/rpmbuild" diff --git a/test/bin/manage_hypervisor_config.sh b/test/bin/manage_hypervisor_config.sh index 6da247db3e..25de888319 100755 --- a/test/bin/manage_hypervisor_config.sh +++ b/test/bin/manage_hypervisor_config.sh @@ -28,17 +28,17 @@ firewall_settings() { sudo firewall-cmd --permanent --zone=libvirt "--${action}-service=mdns" - for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}"; do + for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}"; do if ! sudo virsh net-info "${netname}" &>/dev/null ; then continue fi local vm_bridge - local vm_bridge_cidr vm_bridge=$(sudo virsh net-dumpxml "${netname}" | yq -p xml '.network.bridge.+@name') - vm_bridge_cidr=$(ip addr show "${vm_bridge}" | grep "scope global" | awk '{print $2}') - sudo firewall-cmd --permanent --zone=trusted "--${action}-source"="${vm_bridge_cidr}" + for ip in $(ip addr show "${vm_bridge}" | grep "scope global" | awk '{print $2}'); do + sudo firewall-cmd --permanent --zone=trusted "--${action}-source"="${ip}" + done sudo firewall-cmd --permanent --zone=public "--${action}-port"="${WEB_SERVER_PORT}/tcp" sudo firewall-cmd --reload done @@ -97,6 +97,23 @@ action_create() { sudo ip link add name "${bridge_name}p0" up master "${bridge_name}" type dummy fi + if ! sudo sudo virsh net-info "${VM_DUAL_STACK_NETWORK}" &>/dev/null ; then + local -r dual_stack_netconfig_tmpl="${SCRIPTDIR}/../assets/dual-stack-network.xml" + local -r dual_stack_netconfig_file="${IMAGEDIR}/infra/dual-stack-network.xml" + + mkdir -p "$(dirname "${dual_stack_netconfig_file}")" + envsubst <"${dual_stack_netconfig_tmpl}" >"${dual_stack_netconfig_file}" + + sudo virsh net-define "${dual_stack_netconfig_file}" + sudo virsh net-start "${VM_DUAL_STACK_NETWORK}" + sudo virsh net-autostart "${VM_DUAL_STACK_NETWORK}" + + # Add a dummy port so the bridge is not DOWN and the routing works without + # falling back through the default route + bridge_name=$(sudo virsh net-dumpxml ${VM_DUAL_STACK_NETWORK} | yq -p xml '.network.bridge.+@name') + sudo ip link add name "${bridge_name}p0" up master "${bridge_name}" type dummy + fi + # Firewall firewall_settings "add" } @@ -118,6 +135,13 @@ action_cleanup() { sudo virsh net-undefine "${VM_IPV6_NETWORK}" fi + if sudo virsh net-info "${VM_DUAL_STACK_NETWORK}" &>/dev/null ; then + bridge_name=$(sudo virsh net-dumpxml ${VM_DUAL_STACK_NETWORK} | yq -p xml '.network.bridge.+@name') + sudo ip link del name "${bridge_name}p0" + sudo virsh net-destroy "${VM_DUAL_STACK_NETWORK}" + sudo virsh net-undefine "${VM_DUAL_STACK_NETWORK}" + fi + # Storage pool for pool_name in $(sudo virsh pool-list --name | awk '/vm-storage/ {print $1}') ; do if sudo virsh pool-info "${pool_name}" &>/dev/null; then diff --git a/test/resources/libipv6.py b/test/resources/libipv6.py index 0537018615..7937076a23 100644 --- a/test/resources/libipv6.py +++ b/test/resources/libipv6.py @@ -26,6 +26,10 @@ def must_be_ipv6(addr: str) -> None: BuiltIn().should_be_true(is_ipv6(addr)) +def must_not_be_ipv6(addr: str) -> None: + BuiltIn().should_not_be_true(is_ipv6(addr)) + + def add_brackets_if_ipv6(ip: str) -> str: """ Add square brackets to the given IP if its ipv6 for later use in other tools (e.g. curl) diff --git a/test/scenarios/el94-src@dual-stack.sh b/test/scenarios/el94-src@dual-stack.sh new file mode 100644 index 0000000000..9e4292b720 --- /dev/null +++ b/test/scenarios/el94-src@dual-stack.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +scenario_create_vms() { + prepare_kickstart host1 kickstart.ks.template rhel-9.4-microshift-source + launch_vm host1 rhel-9.4 "${VM_DUAL_STACK_NETWORK}" +} + +scenario_remove_vms() { + remove_vm host1 +} + +scenario_run_tests() { + local -r vmname=$(full_vm_name host1) + local -r vm_ip1=$("${ROOTDIR}/scripts/devenv-builder/manage-vm.sh" ip -n "${vmname}" | head -1) + local -r vm_ip2=$("${ROOTDIR}/scripts/devenv-builder/manage-vm.sh" ip -n "${vmname}" | tail -1) + + run_tests host1 \ + --variable "USHIFT_HOST_IP1:${vm_ip1}" \ + --variable "USHIFT_HOST_IP2:${vm_ip2}" \ + suites/ipv6/dualstack.robot +} diff --git a/test/suites/ipv6/dualstack.robot b/test/suites/ipv6/dualstack.robot new file mode 100644 index 0000000000..cc25672f6f --- /dev/null +++ b/test/suites/ipv6/dualstack.robot @@ -0,0 +1,154 @@ +*** Settings *** +Documentation Tests related to MicroShift running in a dual stack ipv4+6 host + +Resource ../../resources/common.resource +Resource ../../resources/oc.resource +Resource ../../resources/ostree-health.resource +Resource ../../resources/microshift-network.resource +Resource ../../resources/microshift-config.resource +Library ../../resources/libipv6.py + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags ipv6 network + + +*** Variables *** +${USHIFT_HOST_IP1} ${EMPTY} +${USHIFT_HOST_IP2} ${EMPTY} +${HOSTNAME} hello-microshift.dualstack.cluster.local + + +*** Test Cases *** +Verify New Pod Works With IPv6 + [Documentation] Verify IPv6 services are routable. + [Setup] Run Keywords + ... Save Default MicroShift Config + ... Migrate To Dual Stack + ... Create Hello MicroShift Pod + ... Expose Hello MicroShift Service Via Route IPv6 + ... Restart Router + + ${pod_ip}= Oc Get JsonPath pod ${NAMESPACE} hello-microshift .status.podIPs[0].ip + Must Not Be Ipv6 ${pod_ip} + ${pod_ip}= Oc Get JsonPath pod ${NAMESPACE} hello-microshift .status.podIPs[1].ip + Must Be Ipv6 ${pod_ip} + ${service_ip}= Oc Get JsonPath svc ${NAMESPACE} hello-microshift .spec.clusterIP + Must Be Ipv6 ${service_ip} + + Wait Until Keyword Succeeds 10x 6s + ... Access Hello Microshift Success ushift_ip=${USHIFT_HOST_IP1} + ... ushift_port=${HTTP_PORT} + ... hostname=${HOSTNAME} + Wait Until Keyword Succeeds 10x 6s + ... Access Hello Microshift Success ushift_ip=${USHIFT_HOST_IP2} + ... ushift_port=${HTTP_PORT} + ... hostname=${HOSTNAME} + + [Teardown] Run Keywords + ... Delete Hello MicroShift Route + ... Delete Hello MicroShift Pod And Service + ... Wait For Service Deletion With Timeout + ... Restore Default MicroShift Config + ... Restart MicroShift + +Verify New Pod Works With IPv4 + [Documentation] Verify IPv4 services are routable. + [Setup] Run Keywords + ... Save Default MicroShift Config + ... Migrate To Dual Stack + ... Create Hello MicroShift Pod + ... Expose Hello MicroShift Service Via Route IPv4 + ... Restart Router + + ${pod_ip}= Oc Get JsonPath pod ${NAMESPACE} hello-microshift .status.podIPs[0].ip + Must Not Be Ipv6 ${pod_ip} + ${pod_ip}= Oc Get JsonPath pod ${NAMESPACE} hello-microshift .status.podIPs[1].ip + Must Be Ipv6 ${pod_ip} + ${service_ip}= Oc Get JsonPath svc ${NAMESPACE} hello-microshift .spec.clusterIP + Must Not Be Ipv6 ${service_ip} + + Wait Until Keyword Succeeds 10x 6s + ... Access Hello Microshift Success ushift_ip=${USHIFT_HOST_IP1} + ... ushift_port=${HTTP_PORT} + ... hostname=${HOSTNAME} + Wait Until Keyword Succeeds 10x 6s + ... Access Hello Microshift Success ushift_ip=${USHIFT_HOST_IP2} + ... ushift_port=${HTTP_PORT} + ... hostname=${HOSTNAME} + + [Teardown] Run Keywords + ... Delete Hello MicroShift Route + ... Delete Hello MicroShift Pod And Service + ... Wait For Service Deletion With Timeout + ... Restore Default MicroShift Config + ... Restart MicroShift + + +*** Keywords *** +Setup + [Documentation] Test suite setup + Initialize Global Variables + Login MicroShift Host + Setup Suite With Namespace + Wait Until Greenboot Health Check Exited + +Teardown + [Documentation] Test suite teardown + Teardown Suite With Namespace + Logout MicroShift Host + +Initialize Global Variables + [Documentation] Initializes global variables. + Log IP1: ${USHIFT_HOST_IP1} IPv6: ${USHIFT_HOST_IP2} + Should Not Be Empty ${USHIFT_HOST_IP1} USHIFT_HOST_IP1 variable is required + Should Not Be Empty ${USHIFT_HOST_IP2} USHIFT_HOST_IP2 variable is required + +Migrate To Dual Stack + [Documentation] Configure MicroShift to enable dual stack network + + ${dual_stack}= CATENATE SEPARATOR=\n + ... --- + ... network: + ... \ \ clusterNetwork: [10.42.0.0/16, fd01::/48] + ... \ \ serviceNetwork: [10.43.0.0/16, fd02::/112] + ${replaced}= Replace MicroShift Config ${dual_stack} + Upload MicroShift Config ${replaced} + Restart MicroShift + +Delete Hello MicroShift Route + [Documentation] Delete route for cleanup. + Oc Delete route/hello-microshift -n ${NAMESPACE} + +Wait For Service Deletion With Timeout + [Documentation] Polls for service and endpoint by "app=hello-microshift" label. Fails if timeout + ... expires. This check is unique to this test suite because each test here reuses the same namespace. Since + ... the tests reuse the service name, a small race window exists between the teardown of one test and the setup + ... of the next. This produces flakey failures when the service or endpoint names collide. + Wait Until Keyword Succeeds 30s 1s + ... Network APIs With Test Label Are Gone + +Expose Hello MicroShift Service Via Route IPv4 + [Documentation] Expose the "hello microshift" application through the Route + Run With Kubeconfig oc apply -n ${NAMESPACE} -f assets/hello-microshift-service.yaml + Oc Expose svc hello-microshift --hostname ${HOSTNAME} -n ${NAMESPACE} + +Expose Hello MicroShift Service Via Route IPv6 + [Documentation] Expose the "hello microshift" application through the Route + Run With Kubeconfig oc apply -n ${NAMESPACE} -f assets/hello-microshift-service-ipv6.yaml + Oc Expose svc hello-microshift --hostname ${HOSTNAME} -n ${NAMESPACE} + +Network APIs With Test Label Are Gone + [Documentation] Check for service and endpoint by "app=hello-microshift" label. Succeeds if response matches + ... "No resources found in namespace." Fail if not. + ${match_string}= Catenate No resources found in ${NAMESPACE} namespace. + ${match_string}= Remove String ${match_string} " + ${response}= Run With Kubeconfig oc get svc,ep -l app\=hello-microshift -n ${NAMESPACE} + Should Be Equal As Strings ${match_string} ${response} strip_spaces=True + +Restart Router + [Documentation] Restart the router and wait for readiness again. The router is sensitive to apiserver + ... downtime and might need a restart (after the apiserver is ready) to resync all the routes. + Run With Kubeconfig oc rollout restart deployment router-default -n openshift-ingress + Named Deployment Should Be Available router-default openshift-ingress 5m