diff --git a/e2e/main.sh b/e2e/main.sh new file mode 100755 index 0000000000..a5aae51c6b --- /dev/null +++ b/e2e/main.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +OUTPUT_DIR="${ARTIFACT_DIR:-${SCRIPT_DIR}/../_output}/microshift-e2e-$(date +'%Y%m%d-%H%M%S')/" + +usage() { + echo "Usage: $(basename "${0}") {list|run} [filter]" + echo "" + echo " list Lists tests" + echo " run Runs tests" + echo " filter Simple string to match against test files, e.g. 'reboot', 'boot', 'smoke'" + echo "" + echo " Script expects two environmental variables:" + echo " - USHIFT_IP" + echo " - USHIFT_USER" + echo "" + echo " Script assumes following:" + echo " - Passwordless SSH to \$USHIFT_USER@\$USHIFT_IP is configured" + echo " - Both hosts already exchanged their public keys:" + echo " - Test runner has MicroShift's sshd key in ~/.ssh/known_keys" + echo " - Remote \$USHIFT_USER has test runner's key in ~/.ssh/authorized_keys" + echo " - Passwordless sudo for \$USHIFT_USER" + echo "" + echo " Script aims to be target platform agnostic. It means that for some environments (e.g. GCP)" + echo " it might be required to export firewall::open_port and firewall::close_port functions" + echo " so the tests can open custom ports" + + [ -n "$1" ] && echo -e "\nERROR: $1" + exit 1 +} + +log() { + echo -e "$(date +'%H:%M:%S.%N') $*" +} + +ssh_cmd() { + local cmd="${1}" + ssh -o BatchMode=yes "${USHIFT_USER}@${USHIFT_IP}" "${cmd}" +} + +var_should_not_be_empty() { + local var_name=${1} + if [ -z "${!var_name+x}" ]; then + echo >&2 "Environmental variable '${var_name}' is unset" + return 1 + elif [ -z "${!var_name}" ]; then + echo >&2 "Environmental variable '${var_name}' is empty" + return 1 + fi +} + +function_should_be_exported() { + local fname=${1} + if ! declare -F "${fname}"; then + log "WARNING: Function '${fname}' is unexported. It is expected that function is provided for interacting with cloud provider" + return 1 + fi +} + +check_passwordless_ssh() { + ssh_cmd "true" || { + echo "Failed to access ${USHIFT_IP}:" + echo " - Test runner should have MicroShift's sshd key in ~/.ssh/known_keys" + echo " - Remote \$USHIFT_USER should have test runner's key in ~/.ssh/authorized_keys" + exit 1 + } +} + +check_passwordless_sudo() { + ssh_cmd "sudo --non-interactive true" || { + echo "Failed to run sudo command as ${USHIFT_USER} without password" + exit 1 + } +} + +prechecks() { + var_should_not_be_empty USHIFT_IP && var_should_not_be_empty USHIFT_USER || exit 1 + check_passwordless_ssh + check_passwordless_sudo + + # Just warning for now + # Following functions needed only for runs in CI + function_should_be_exported firewall::open_port || true + function_should_be_exported firewall::close_port || true +} + +microshift_get_konfig() { + tmpfile=$(mktemp /tmp/microshift-e2e-konfig.XXXXXX) + ssh_cmd 'sudo cat /var/lib/microshift/resources/kubeadmin/'"${USHIFT_IP}"'/kubeconfig' >"${tmpfile}" + echo "${tmpfile}" +} + +microshift_check_readiness() { + local output_dir="${1}" + log "Waiting for MicroShift to become ready" + ssh_cmd "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" &>"${output_dir}/0002-readiness-check.log" +} + +microshift_setup() { + local output_dir="${1}" + log "Setting up and starting MicroShift" + ssh_cmd 'cat << EOF | sudo tee /etc/microshift/config.yaml +--- +apiServer: + subjectAltNames: + - '"${USHIFT_IP}"' +EOF' &>"${output_dir}/0001-setup.log" + ssh_cmd "sudo systemctl enable --now microshift" &>>"${output_dir}/0001-setup.log" +} + +microshift_debug_info() { + local output_dir="${1}" + log "Gathering debug info to ${output_dir}/0020-cluster-debug-info.log" + scp "${SCRIPT_DIR}/../validate-microshift/cluster-debug-info.sh" "${USHIFT_USER}@${USHIFT_IP}:/tmp/cluster-debug-info.sh" + ssh_cmd "sudo /tmp/cluster-debug-info.sh" &>"${output_dir}/0020-cluster-debug-info.log" +} + +microshift_cleanup() { + local output_dir="${1}" + log "Cleaning MicroShift" + ssh_cmd "echo 1 | sudo microshift-cleanup-data --all" &>"${output_dir}/0000-cleanup.log" +} + +microshift_reprovision() { + local output_dir="$1" + + prep_start=$(date +%s) + microshift_cleanup "${output_dir}" + microshift_setup "${output_dir}" + microshift_check_readiness "${output_dir}" + prep_dur=$(($(date +%s) - prep_start)) + log "Reprovisioning took $((prep_dur / 60))m $((prep_dur % 60))s." +} + +microshift_health_summary() { + log "Summary of MicroShift health" + + # Because test might be "destructive" (i.e. tear down and set up again MicroShift) + # so these commands are executed via ssh. + # Alternative is to copy kubeconfig second time in the same test. + ssh_cmd "mkdir -p ~/.kube/ && sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config ; \ + oc get pods -A ; \ + oc get nodes -o wide ; \ + oc get events -A --sort-by=.metadata.creationTimestamp | head -n 20" +} + +run_test() { + local test=$1 + local output=$2 + log "${test} - RUNNING" + + konfig=$(microshift_get_konfig) + trap 'rm -f "${konfig}"' RETURN + + test_start=$(date +%s) + set +e + KUBECONFIG="${konfig}" "${SCRIPT_DIR}/tests/${test}" &>"${output}/0010-test.log" + res=$? + set -e + test_dur=$(($(date +%s) - test_start)) + + if [ ${res} -eq 0 ]; then + log "${test} - SUCCESS after $((test_dur / 60))m $((test_dur % 60))s." + return 0 + fi + + log "${test} - FAILURE after $((test_dur / 60))m $((test_dur % 60))s." + microshift_health_summary || true + microshift_debug_info "${output}" || true + return 1 +} + +list() { + local -r filter="*${1:-}*.sh" + find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "${filter}" -printf "%f\n" | sort +} + +run() { + local -r to_run=$(list "${1}") + + prechecks + log "Following tests will run:\n${to_run}" + [ ! -d "${OUTPUT_DIR}" ] && mkdir -p "${OUTPUT_DIR}" + + testsuite_start=$(date +%s) + microshift_reprovision "${OUTPUT_DIR}" + + all_successful=true + reprovision=false + for t in ${to_run}; do + local tout="${OUTPUT_DIR}/${t}/" + mkdir -p "${tout}" + + if "${reprovision}"; then + log "Reprovisioning MicroShift before next test" + microshift_reprovision "${tout}" + fi + + run_test "${t}" "${tout}" || all_successful=false + + if grep -q "reprovision_after_test=true" "${SCRIPT_DIR}/tests/${t}"; then + reprovision=true + fi + done + + testsuite_dur=$(($(date +%s) - testsuite_start)) + log "MicroShift E2E took $((testsuite_dur / 60))m $((testsuite_dur % 60))s." + "${all_successful}" +} + +[ $# -eq 0 ] && { + usage "Missing expected arguments" +} + +cmd="$1" +shift +"${cmd}" "${1:-}" diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh new file mode 100755 index 0000000000..943f87a1ec --- /dev/null +++ b/e2e/tests/0030-router-smoke-test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# reprovision_after_test=false + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +RETRIES=3 +BACKOFF=3s + +# shellcheck disable=SC2317 # Don't warn about unreachable commands in this function +cleanup() { + oc delete route hello-microshift || true + oc delete service hello-microshift || true + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true + if declare -F firewall::close_port; then firewall::close_port 80 tcp || true; fi +} +trap 'cleanup' EXIT + +declare -F firewall::open_port && firewall::open_port 80 tcp + +oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +oc expose pod hello-microshift +oc expose svc hello-microshift --hostname hello-microshift.cluster.local +oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s + +for _ in $(seq "${RETRIES}"); do + set +e + response=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) + result=$? + set -e + + [ ${result} -eq 0 ] && + echo "${response}" | grep -q -E "HTTP.*200" && + echo "${response}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${BACKOFF}" +done +exit 1 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh new file mode 100755 index 0000000000..1e123cdf56 --- /dev/null +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# reprovision_after_test=false + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +RETRIES=3 +BACKOFF=3s + +# shellcheck disable=SC2317 # Don't warn about unreachable commands in this function +cleanup() { + oc delete service hello-microshift || true + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true + if declare -F firewall::close_port; then firewall::close_port 5678 tcp || true; fi +} +trap 'cleanup' EXIT + +declare -F firewall::open_port && firewall::open_port 5678 tcp +oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +oc create service loadbalancer hello-microshift --tcp=5678:8080 +oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s + +for _ in $(seq "${RETRIES}"); do + set +e + response=$(curl -i "${USHIFT_IP}":5678 2>&1) + result=$? + set -e + + [ ${result} -eq 0 ] && + echo "${response}" | grep -q -E "HTTP.*200" && + echo "${response}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${BACKOFF}" +done +exit 1 diff --git a/e2e/tests/0040-reboot.sh b/e2e/tests/0040-reboot.sh new file mode 100755 index 0000000000..22a5b84897 --- /dev/null +++ b/e2e/tests/0040-reboot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# reprovision_after_test=false + +set -euo pipefail +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +RETRIES=5 +BACKOFF=20 + +wait_until() { + local cmd=$* + for _ in $(seq "${RETRIES}"); do + ${cmd} && return 0 + sleep "${BACKOFF}" + done + return 1 +} + +cleanup() { + oc delete -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" +} +trap 'cleanup' EXIT + +oc create -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" +oc wait --for=condition=Ready --timeout=120s pod/test-pod + +set +e +ssh -v "${USHIFT_USER}@${USHIFT_IP}" "sudo reboot now" +res=$? +set -e + +# Allow for `ssh` command errors (255 exit code) like "connection closed by remote host" +# Fail on other errors (coming from the command executed remotely itself) +if [ "${res}" -ne 0 ] && [ "${res}" -ne 255 ]; then + exit 1 +fi + +wait_until ssh "${USHIFT_USER}@${USHIFT_IP}" "true" +ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" +oc wait --for=condition=Ready --timeout=120s pod/test-pod diff --git a/e2e/tests/0100-greenboot.sh b/e2e/tests/0100-greenboot.sh new file mode 100755 index 0000000000..be96e4ca80 --- /dev/null +++ b/e2e/tests/0100-greenboot.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# reprovision_after_test=true + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +scp "${SCRIPT_PATH}/../../docs/config/busybox_running_check.sh" "${SCRIPT_PATH}/assets/greenboot-test.sh" "${USHIFT_USER}@${USHIFT_IP}":/tmp/ +ssh -q "${USHIFT_USER}@${USHIFT_IP}" "chmod +x /tmp/greenboot-test.sh /tmp/busybox_running_check.sh && sudo /tmp/greenboot-test.sh" diff --git a/e2e/tests/assets/greenboot-test.sh b/e2e/tests/assets/greenboot-test.sh new file mode 100644 index 0000000000..ef9c1061be --- /dev/null +++ b/e2e/tests/assets/greenboot-test.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +function check_greenboot_exit_status() { + local expectedRC=$1 + local cleanup=$2 + + if [ "${cleanup}" -ne 0 ]; then + echo 1 | microshift-cleanup-data --all + systemctl enable --now microshift || true + fi + + for check_script in $(find /etc/greenboot/check/ -name \*.sh | sort); do + echo Running "${check_script}"... + local currentRC=1 + if ${check_script}; then + currentRC=0 + fi + + if [ ${currentRC} != "${expectedRC}" ]; then + exit 1 + fi + done +} + +# +# Initial check must succeed (set timeout of 180s to speed up the process) +# +tee /etc/greenboot/greenboot.conf &>/dev/null </dev/null </dev/null </dev/null <