From f29680179b0889645c16c49b0ef3131f64533cb5 Mon Sep 17 00:00:00 2001 From: Chris Suszynski Date: Mon, 2 Nov 2020 11:21:44 +0100 Subject: [PATCH] Backport of #530 Eventing upgrade tests * Splitting upgrade test to accommodate Eventing * Add an image template * Upgrade tests with Eventing by default * Proper error reporting from shell scripts * Remove serving & eventing ns at teardown * Adding missing return statements + prober end idempotency * Declare a waited PIDs array * Rely on PID directory instead of Bash arrays * Eventing test bits deploy properly * Proper waiting on prober's end * Creating a namespaces for previous release * Add env vars to configure upgrade tests * Lowering interval from 2ms to 50ms To avoid loosing events and most notably "finished" event, I'm lowering interval between sent event to 50 ms from previously used 2 ms. --- hack/lib/catalogsource.bash | 10 +- hack/lib/common.bash | 36 ++++-- hack/lib/namespaces.bash | 12 +- hack/lib/serverless.bash | 64 +++++++---- hack/lib/vars.bash | 3 +- test/eventing.bash | 113 ++++++++++++++++++- test/lib.bash | 216 +++++++++++++++++++++++++++++++----- test/serving.bash | 176 ++++++++++++++++------------- test/upstream-e2e-tests.sh | 16 +-- 9 files changed, 490 insertions(+), 156 deletions(-) diff --git a/hack/lib/catalogsource.bash b/hack/lib/catalogsource.bash index 8310f5f725..ba367e242d 100644 --- a/hack/lib/catalogsource.bash +++ b/hack/lib/catalogsource.bash @@ -12,7 +12,9 @@ function ensure_catalogsource_installed { function install_catalogsource { logger.info "Installing CatalogSource" - local rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")" + local rootdir pull_user + + rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")" # Add a user that is allowed to pull images from the registry. pull_user="puller" @@ -36,8 +38,8 @@ function install_catalogsource { cat "$csv" # Build the bundle image in the cluster-internal registry. - oc -n "$OLM_NAMESPACE" new-build --binary --strategy=docker --name serverless-bundle - oc -n "$OLM_NAMESPACE" start-build serverless-bundle --from-dir olm-catalog/serverless-operator -F + oc -n "$OLM_NAMESPACE" new-build --binary --strategy=docker --name serverless-bundle || return $? + oc -n "$OLM_NAMESPACE" start-build serverless-bundle --from-dir olm-catalog/serverless-operator -F || return $? # Undo potential changes to the CSV to not pollute the repository. mv "${rootdir}/bkp.yaml" "$csv" @@ -48,7 +50,7 @@ function install_catalogsource { # Install the index deployment. # This image was built using the Dockerfile at 'olm-catalog/serverless-operator/index.Dockerfile'. - cat </dev/null 2>&1; then + oc create namespace "${ns}" + fi +} + function create_namespaces { logger.info 'Create namespaces' for ns in "${NAMESPACES[@]}"; do - if ! oc get ns "${ns}" >/dev/null 2>&1; then - oc create ns "${ns}" - fi + ensure_namespace "${ns}" done # Create an OperatorGroup if there are no other ones in the namespace. if [[ $(oc get operatorgroups -oname -n "${OPERATORS_NAMESPACE}" | wc -l) -eq 0 ]]; then diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 173d253429..c4b1e02324 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -23,26 +23,30 @@ function ensure_serverless_installed { } function install_serverless_previous { - local rootdir - rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")" + logger.info "Installing previous version of Serverless..." # Remove installplan from previous installations, leaving this would make the operator # upgrade to the latest version immediately remove_installplan "$CURRENT_CSV" deploy_serverless_operator "$PREVIOUS_CSV" || return $? - deploy_knativeserving_cr || return $? -} -function remove_installplan { - local install_plan - install_plan=$(find_install_plan $1) - if [[ -n $install_plan ]]; then - oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}" + # TODO(ksuszyns): Remove this if block if no longer required + if versions.le "$(metadata.get olm.replaces)" 1.11.0; then + logger.info "Ensure ${SERVING_NAMESPACE} and ${EVENTING_NAMESPACE} \ +namespaces exists, as ${PREVIOUS_CSV} didn't created them automatically." + + ensure_namespace "${SERVING_NAMESPACE}" + ensure_namespace "${EVENTING_NAMESPACE}" fi + + deploy_knativeserving_cr || return $? + deploy_knativeeventing_cr || return $? + logger.success "Previous version of Serverless is installed: $PREVIOUS_CSV" } function install_serverless_latest { + logger.info "Installing latest version of Serverless..." deploy_serverless_operator_latest || return $? if [[ $INSTALL_SERVING == "true" ]]; then @@ -54,20 +58,32 @@ function install_serverless_latest { if [[ $INSTALL_KAFKA == "true" ]]; then deploy_knativekafka_cr || return $? fi + + logger.success "Latest version of Serverless is installed: $CURRENT_CSV" +} + +function remove_installplan { + local install_plan csv + csv="${1:?Pass a CSV as arg[1]}" + logger.info "Removing installplan for $csv" + install_plan=$(find_install_plan "$csv") + if [[ -n $install_plan ]]; then + oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}" + else + logger.debug "No install plan for $csv" + fi } function deploy_serverless_operator_latest { - local rootdir - rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")" deploy_serverless_operator "$CURRENT_CSV" } function deploy_serverless_operator { - local csv - csv="$1" - logger.info "Install the Serverless Operator ${csv}" - - cat < "$tmpfile" </dev/null 2>&1; then + oc delete namespace "${SERVING_NAMESPACE}" + fi if oc get knativeeventing.operator.knative.dev knative-eventing -n "${EVENTING_NAMESPACE}" >/dev/null 2>&1; then logger.info 'Removing KnativeEventing CR' @@ -208,6 +229,9 @@ function teardown_serverless { fi logger.info 'Ensure no knative eventing or knative kafka pods running' timeout 600 "[[ \$(oc get pods -n ${EVENTING_NAMESPACE} --field-selector=status.phase!=Succeeded -o jsonpath='{.items}') != '[]' ]]" || return 10 + if oc get namespace "${EVENTING_NAMESPACE}" >/dev/null 2>&1; then + oc delete namespace "${EVENTING_NAMESPACE}" + fi oc delete subscriptions.operators.coreos.com -n "${OPERATORS_NAMESPACE}" "${OPERATOR}" 2>/dev/null for ip in $(oc get installplan -n "${OPERATORS_NAMESPACE}" | grep serverless-operator | cut -f1 -d' '); do diff --git a/hack/lib/vars.bash b/hack/lib/vars.bash index 0e8929fd9b..d20281f356 100644 --- a/hack/lib/vars.bash +++ b/hack/lib/vars.bash @@ -53,7 +53,8 @@ export INSTALL_PREVIOUS_VERSION="${INSTALL_PREVIOUS_VERSION:-"false"}" # Using first channel on the list, instead of default one -export OLM_CHANNEL="${OLM_CHANNEL:-$(metadata.get 'olm.channels.list[*]' | head -n 1)}" +OLM_CHANNEL="${OLM_CHANNEL:-$(metadata.get 'olm.channels.list[*]' | head -n 1)}" +export OLM_CHANNEL # Change this when upgrades need switching to a different channel export OLM_UPGRADE_CHANNEL="${OLM_UPGRADE_CHANNEL:-"$OLM_CHANNEL"}" export OLM_SOURCE="${OLM_SOURCE:-"$OPERATOR"}" diff --git a/test/eventing.bash b/test/eventing.bash index 9df32e769d..e3b82f8898 100644 --- a/test/eventing.bash +++ b/test/eventing.bash @@ -1,15 +1,18 @@ #!/usr/bin/env bash +readonly EVENTING_READY_FILE="/tmp/eventing-prober-ready" +readonly EVENTING_PROBER_FILE="/tmp/eventing-prober-signal" + function upstream_knative_eventing_e2e { - ( logger.info 'Running eventing tests' local failed=0 - TEST_IMAGE_TEMPLATE="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}" + export TEST_IMAGE_TEMPLATE="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}" - cd "$KNATIVE_EVENTING_HOME" || return $? + cd "${KNATIVE_EVENTING_HOME}" || return $? + # shellcheck disable=SC1090 source "${KNATIVE_EVENTING_HOME}/openshift/e2e-common.sh" # run_e2e_tests defined in knative-eventing @@ -18,5 +21,107 @@ function upstream_knative_eventing_e2e { print_test_result ${failed} return $failed - ) +} + +function actual_eventing_version { + oc get knativeeventing.operator.knative.dev \ + knative-eventing -n "${EVENTING_NAMESPACE}" -o=jsonpath="{.status.version}" \ + || return $? +} + +function prepare_knative_eventing_tests { + logger.info 'Nothing to prepare for Eventing upgrade tests' +} + +function run_eventing_preupgrade_test { + logger.info 'Running Eventing pre upgrade tests' + + cd "${KNATIVE_EVENTING_HOME}" || return $? + + local image_template + # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image + image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}" + + go_test_e2e -tags=preupgrade \ + -timeout=10m ./test/upgrade \ + --imagetemplate="${image_template}" \ + || return $? + + logger.success 'Eventing pre upgrade tests passed' +} + +function start_eventing_prober { + local eventing_prober_pid pid_file image_template eventing_prober_interval + pid_file="${1:?Pass a PID file as arg[1]}" + logger.info 'Starting Eventing prober' + + EVENTING_PROBER_INTERVAL_MSEC="${EVENTING_PROBER_INTERVAL_MSEC:-50}" + eventing_prober_interval="${EVENTING_PROBER_INTERVAL_MSEC}ms" + + + rm -fv "${EVENTING_PROBER_FILE}" "${EVENTING_READY_FILE}" + cd "${KNATIVE_EVENTING_HOME}" || return $? + + # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image + image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}" + + # FIXME: knative/operator#297 Restore scale to zero setting + E2E_UPGRADE_TESTS_SERVING_SCALETOZERO=false \ + E2E_UPGRADE_TESTS_SERVING_USE=true \ + E2E_UPGRADE_TESTS_CONFIGMOUNTPOINT=/.config/wathola \ + E2E_UPGRADE_TESTS_INTERVAL="${eventing_prober_interval}" \ + go_test_e2e -tags=probe \ + -timeout=30m \ + ./test/upgrade \ + --pipefile="${EVENTING_PROBER_FILE}" \ + --readyfile="${EVENTING_READY_FILE}" \ + --imagetemplate="${image_template}" & + eventing_prober_pid=$! + + logger.debug "Eventing prober PID is ${eventing_prober_pid}" + + echo ${eventing_prober_pid} > "${pid_file}" +} + +function wait_for_eventing_prober_ready { + wait_for_file "${EVENTING_READY_FILE}" || return $? + + logger.success 'Eventing prober is ready' +} + +function end_eventing_prober { + local prober_pid + prober_pid="${1:?Pass a prober pid as arg[1]}" + + end_prober 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}" || return $? +} + +function check_eventing_upgraded { + local latest_version + latest_version="${1:?Pass a target eventing version as arg[1]}" + + logger.debug 'Check KnativeEventing has the latest version with Ready status' + timeout 300 "[[ ! ( \$(oc get knativeeventing.operator.knative.dev \ + knative-eventing -n ${EVENTING_NAMESPACE} -o=jsonpath='{.status.version}') \ + == ${latest_version} && \$(oc get knativeeventing.operator.knative.dev \ + knative-eventing -n ${EVENTING_NAMESPACE} \ + -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" \ + || return $? +} + +function run_eventing_postupgrade_test { + logger.info 'Running Eventing post upgrade tests' + local image_template + + cd "${KNATIVE_EVENTING_HOME}" || return $? + + # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image + image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}" + + go_test_e2e -tags=postupgrade \ + -timeout=10m ./test/upgrade \ + --imagetemplate="${image_template}" \ + || return $? + + logger.success 'Eventing post upgrade tests passed' } diff --git a/test/lib.bash b/test/lib.bash index 7c4a7fe833..8607a0afe3 100644 --- a/test/lib.bash +++ b/test/lib.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# == Overrides & test releated +# == Overrides & test related # shellcheck disable=SC1091,SC1090 source "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")/hack/lib/__sources__.bash" @@ -58,11 +58,11 @@ function serverless_operator_e2e_tests { go_test_e2e -failfast -tags=e2e -timeout=30m -parallel=1 ./test/e2e \ --channel "$OLM_CHANNEL" \ --kubeconfigs "${kubeconfigs_str}" \ - "$@" || failed=1 + "$@" || failed=$? print_test_result ${failed} - wait_for_knative_serving_ingress_ns_deleted || return 1 + wait_for_knative_serving_ingress_ns_deleted || return $? return $failed } @@ -83,7 +83,7 @@ function serverless_operator_kafka_e2e_tests { go_test_e2e -failfast -tags=e2e -timeout=30m -parallel=1 ./test/e2ekafka \ --channel "$OLM_CHANNEL" \ --kubeconfigs "${kubeconfigs_str}" \ - "$@" || failed=1 + "$@" || failed=$? print_test_result ${failed} @@ -109,7 +109,7 @@ function downstream_serving_e2e_tests { go_test_e2e -failfast -timeout=30m -parallel=1 ./test/servinge2e \ --kubeconfig "${kubeconfigs[0]}" \ --kubeconfigs "${kubeconfigs_str}" \ - "$@" || failed=1 + "$@" || failed=$? print_test_result ${failed} @@ -132,7 +132,7 @@ function downstream_knative_kafka_e2e_tests { go_test_e2e -failfast -timeout=30m -parallel=1 ./test/extensione2e/kafka \ --kubeconfig "${kubeconfigs[0]}" \ --kubeconfigs "${kubeconfigs_str}" \ - "$@" || failed=1 + "$@" || failed=$? print_test_result ${failed} @@ -156,19 +156,180 @@ function downstream_eventing_e2e_tests { go_test_e2e -failfast -timeout=30m -parallel=1 ./test/eventinge2e \ --kubeconfig "${kubeconfigs[0]}" \ --kubeconfigs "${kubeconfigs_str}" \ - "$@" || failed=1 + "$@" || failed=$? print_test_result ${failed} return $failed } -function end_prober_test { - local PROBER_PID=$1 - echo "done" > /tmp/prober-signal - logger.info "Waiting for prober test to finish" - wait "${PROBER_PID}" - return $? +# == Upgrade testing + +function run_rolling_upgrade_tests { + logger.info "Running rolling upgrade tests" + + local latest_cluster_version latest_serving_version latest_eventing_version \ + rootdir scope serving_in_scope eventing_in_scope serving_prober_pid \ + eventing_prober_pid prev_serving_version prev_eventing_version \ + ocp_target_version retcode + + scope="${1:?Provide an upgrade scope as arg[1]}" + serving_in_scope="$(echo "${scope}" | grep -vq serving ; echo "$?")" + eventing_in_scope="$(echo "${scope}" | grep -vq eventing ; echo "$?")" + + prev_serving_version="$(actual_serving_version)" + prev_eventing_version="$(actual_eventing_version)" + + # Save the rootdir before changing dir + rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")" + + if (( eventing_in_scope )); then + prepare_knative_eventing_tests || return $? + fi + if (( serving_in_scope )); then + prepare_knative_serving_tests || return $? + fi + + logger.info 'Testing with pre upgrade tests' + + if (( serving_in_scope )); then + run_serving_preupgrade_test || return $? + fi + if (( eventing_in_scope )); then + run_eventing_preupgrade_test || return $? + fi + + logger.info 'Starting prober tests' + + if (( serving_in_scope )); then + start_serving_prober "${prev_serving_version}" /tmp/prober-pid + retcode=$? + serving_prober_pid=$(cat /tmp/prober-pid) + if (( retcode )); then + return $retcode + fi + fi + if (( eventing_in_scope )); then + start_eventing_prober /tmp/prober-pid + retcode=$? + eventing_prober_pid=$(cat /tmp/prober-pid) + if (( retcode )); then + return $retcode + fi + fi + + if (( serving_in_scope )); then + wait_for_serving_prober_ready || return $? + fi + if (( eventing_in_scope )); then + wait_for_eventing_prober_ready || return $? + fi + + if [[ $UPGRADE_SERVERLESS == true ]]; then + latest_serving_version="${KNATIVE_SERVING_VERSION/v/}" + latest_eventing_version="${KNATIVE_EVENTING_VERSION/v/}" + + logger.info "Updating Serverless to ${CURRENT_CSV}" + logger.debug "Serving version: ${prev_serving_version} -> ${latest_serving_version}" + logger.debug "Eventing version: ${prev_eventing_version} -> ${latest_eventing_version}" + + approve_csv "$CURRENT_CSV" "$OLM_UPGRADE_CHANNEL" + if (( serving_in_scope )); then + check_serving_upgraded "${latest_serving_version}" || return $? + fi + if (( eventing_in_scope )); then + check_eventing_upgraded "${latest_eventing_version}" || return $? + fi + fi + + # Might not work in OpenShift CI but we want it here so that we can consume + # this script later and re-use + if [[ $UPGRADE_CLUSTER == true ]]; then + # End the prober test now before we start cluster upgrade, up until now we + # should have zero failed requests. Cluster upgrade will fail probers as + # stuff is moved around. + if (( serving_in_scope )); then + end_serving_prober "${serving_prober_pid}" || return $? + fi + if (( eventing_in_scope )); then + end_eventing_prober "${eventing_prober_pid}" || return $? + fi + + upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}" || return $? + fi + + if (( serving_in_scope )); then + wait_for_serving_test_services_settle || return $? + fi + + logger.info "Running postupgrade tests" + + if (( serving_in_scope )); then + run_serving_postupgrade_test || return $? + fi + if (( eventing_in_scope )); then + run_eventing_postupgrade_test || return $? + fi + + if (( serving_in_scope )); then + end_serving_prober "${serving_prober_pid}" || return $? + fi + if (( eventing_in_scope )); then + end_eventing_prober "${eventing_prober_pid}" || return $? + fi + + cleanup_serving_test_services || return $? + + cd "$rootdir" || return $? + return 0 +} + +function end_prober { + local prober_pid prober_signal retcode title piddir + title=${1:?Pass a title as arg[1]} + prober_pid=${2:?Pass a pid as a arg[2]} + prober_signal=${3:-/tmp/prober-signal} + piddir="${piddir:-/tmp/svls-probes/$$}" + + mkdir -p "${piddir}" || return $? + + if [ -f "${piddir}/${prober_pid}" ]; then + logger.info "Prober of PID ${prober_pid} is closed already." + return 0 + fi + logger.info "Waiting for ${title} prober test to finish" + echo 'done' > "${prober_signal}" + + wait "${prober_pid}" + retcode=$? + echo 'done' > "${piddir}/${prober_pid}" + if ! (( retcode )); then + logger.success "${title} prober passed" + else + logger.error "${title} prober failed" + fi + return $retcode +} + +function upgrade_ocp_cluster { + local ocp_target_version upgrade_ocp_image latest_cluster_version + upgrade_ocp_image="${1:-}" + + if [[ -n "$upgrade_ocp_image" ]]; then + ocp_target_version="$upgrade_ocp_image" + oc adm upgrade --to-image="${UPGRADE_OCP_IMAGE}" \ + --force=true --allow-explicit-upgrade || return $? + else + latest_cluster_version=$(oc adm upgrade | sed -ne '/VERSION/,$ p' \ + | grep -v VERSION | awk '{print $1}' | sort -r | head -n 1) + [[ $latest_cluster_version != "" ]] || return 1 + ocp_target_version="$latest_cluster_version" + oc adm upgrade --to-latest=true --force=true || return $? + fi + timeout 7200 "[[ \$(oc get clusterversion version -o jsonpath='{.status.history[?(@.image==\"${ocp_target_version}\")].state}') != Completed ]]" || return $? + + logger.success "New cluster version: $(oc get clusterversion \ + version -o jsonpath='{.status.desired.version}')" } function teardown { @@ -231,28 +392,28 @@ function create_htpasswd_users { logger.info 'Add users to htpasswd' for i in $(seq 1 $num_users); do - htpasswd -b users.htpasswd "user${i}" "password${i}" + htpasswd -b users.htpasswd "user${i}" "password${i}" || return $? done kubectl create secret generic htpass-secret \ --from-file=htpasswd="$(pwd)/users.htpasswd" \ -n openshift-config \ - --dry-run -o yaml | kubectl apply -f - - oc apply -f openshift/identity/htpasswd.yaml + --dry-run=client -o yaml | kubectl apply -f - || return $? + oc apply -f openshift/identity/htpasswd.yaml || return $? logger.info 'Generate kubeconfig for each user' for i in $(seq 1 $num_users); do cp "${KUBECONFIG}" "user${i}.kubeconfig" occmd="bash -c '! oc login --kubeconfig=user${i}.kubeconfig --username=user${i} --password=password${i} > /dev/null'" - timeout 180 "${occmd}" || return 1 + timeout 180 "${occmd}" || return $? done } function add_roles { logger.info "Adding roles to users" - oc adm policy add-role-to-user admin user1 -n "$TEST_NAMESPACE" - oc adm policy add-role-to-user edit user2 -n "$TEST_NAMESPACE" - oc adm policy add-role-to-user view user3 -n "$TEST_NAMESPACE" + oc adm policy add-role-to-user admin user1 -n "$TEST_NAMESPACE" || return $? + oc adm policy add-role-to-user edit user2 -n "$TEST_NAMESPACE" || return $? + oc adm policy add-role-to-user view user3 -n "$TEST_NAMESPACE" || return $? } function delete_users { @@ -262,10 +423,10 @@ function delete_users { logger.debug "htpasswd user line: ${line}" user=$(echo "${line}" | cut -d: -f1) if [ -f "${user}.kubeconfig" ]; then - rm -v "${user}.kubeconfig" + rm -fv "${user}.kubeconfig" || return $? fi done < "users.htpasswd" - rm -v users.htpasswd + rm -fv users.htpasswd || return $? } function add_systemnamespace_label { @@ -274,8 +435,8 @@ function add_systemnamespace_label { } function add_networkpolicy { - local NAMESPACE=$1 - cat </dev/null 2>&1; then echo -e "\nNew leader Controller has been elected" diff --git a/test/serving.bash b/test/serving.bash index 9ddd47baff..0f9a12dd14 100644 --- a/test/serving.bash +++ b/test/serving.bash @@ -12,6 +12,10 @@ function wait_for_knative_serving_ingress_ns_deleted { } function prepare_knative_serving_tests { + logger.debug 'Preparing Serving tests' + + cd "$KNATIVE_SERVING_HOME" || return $? + # Don't bother with the chaosduck downstream for now rm -rf test/config/chaosduck.yaml @@ -32,8 +36,6 @@ function prepare_knative_serving_tests { function upstream_knative_serving_e2e_and_conformance_tests { logger.info "Running Serving E2E and conformance tests" - ( - cd "$KNATIVE_SERVING_HOME" || return $? prepare_knative_serving_tests || return $? @@ -100,110 +102,126 @@ function upstream_knative_serving_e2e_and_conformance_tests { print_test_result ${failed} return $failed - ) } -function run_knative_serving_rolling_upgrade_tests { - logger.info "Running Serving rolling upgrade tests" - ( - local failed upgrade_to latest_cluster_version cluster_version prev_serving_version latest_serving_version +function actual_serving_version { + oc get knativeserving.operator.knative.dev \ + knative-serving -n "${SERVING_NAMESPACE}" -o=jsonpath="{.status.version}" \ + || return $? +} - # Save the rootdir before changing dir - rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")" +function run_serving_preupgrade_test { + logger.info 'Running Serving pre upgrade tests' - cd "$KNATIVE_SERVING_HOME" || return $? + local image_template - prepare_knative_serving_tests || return $? + cd "${KNATIVE_SERVING_HOME}" || return $? - failed=0 image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}" - PROBE_FRACTION=1.0 - prev_serving_version=$(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.version}") - - if [[ ${prev_serving_version} < "0.14.0" ]]; then - PROBE_FRACTION=0.95 - fi - logger.info "Target success fraction is $PROBE_FRACTION" go_test_e2e -tags=preupgrade -timeout=20m ./test/upgrade \ --imagetemplate "$image_template" \ --kubeconfig "$KUBECONFIG" \ - --resolvabledomain || return 1 + --resolvabledomain || return $? - logger.info "Starting prober test" + logger.success 'Serving pre upgrade tests passed' +} + +function start_serving_prober { + local image_template prev_serving_version probe_fraction serving_prober_pid \ + pid_file + prev_serving_version="${1:?Pass a previous Serving version as arg[1]}" + pid_file="${2:?Pass a PID file as arg[2]}" + + logger.info 'Starting Serving prober' + + rm -fv /tmp/prober-signal + cd "${KNATIVE_SERVING_HOME}" || return $? + + probe_fraction=1.0 + if [[ ${prev_serving_version} < "0.14.0" ]]; then + probe_fraction=0.95 + fi + logger.info "Target success fraction for Serving is ${probe_fraction}" + + image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}" - rm -f /tmp/prober-signal - go_test_e2e -tags=probe -timeout=20m ./test/upgrade \ - -probe.success_fraction=$PROBE_FRACTION \ + go_test_e2e -tags=probe \ + -timeout=30m \ + ./test/upgrade \ + -probe.success_fraction=${probe_fraction} \ --imagetemplate "$image_template" \ --kubeconfig "$KUBECONFIG" \ --resolvabledomain & + serving_prober_pid=$! + logger.debug "Serving prober PID is ${serving_prober_pid}" + + echo ${serving_prober_pid} > "${pid_file}" +} + +function wait_for_serving_prober_ready { # Wait for the upgrade-probe kservice to be ready before proceeding - timeout 900 '[[ $(oc get services.serving.knative.dev upgrade-probe -n serving-tests -o=jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}") != True ]]' || return 1 - - PROBER_PID=$! - - if [[ $UPGRADE_SERVERLESS == true ]]; then - latest_serving_version=$(echo $KNATIVE_SERVING_VERSION | sed "s/v//") - - logger.info "updating serving version from ${prev_serving_version} to ${latest_serving_version}" - - # Get latest CSV from the given channel - upgrade_to="$CURRENT_CSV" - - cluster_version=$(oc get clusterversion -o=jsonpath="{.items[0].status.history[?(@.state==\"Completed\")].version}") - if [[ "$cluster_version" = 4.1.* || "${HOSTNAME}" = *ocp-41* || \ - "$cluster_version" = 4.2.* || "${HOSTNAME}" = *ocp-42* ]]; then - if approve_csv "$upgrade_to" "$OLM_UPGRADE_CHANNEL" ; then # Upgrade should fail on OCP 4.1, 4.2 - return 1 - fi - # Check we got RequirementsNotMet error - [[ $(oc get ClusterServiceVersion $upgrade_to -n $OPERATORS_NAMESPACE -o=jsonpath="{.status.requirementStatus[?(@.name==\"$upgrade_to\")].message}") =~ "requirement not met: minKubeVersion" ]] || return 1 - # Check KnativeServing still has the old version - [[ $(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.version}") == "$prev_serving_version" ]] || return 1 - else - approve_csv "$upgrade_to" "$OLM_UPGRADE_CHANNEL" || return 1 - # Check KnativeServing has the latest version with Ready status - timeout 300 '[[ ! ( $(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.version}") == $latest_serving_version && $(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}") == True ) ]]' || return 1 - fi - end_prober_test ${PROBER_PID} || return $? - fi + timeout 900 "[[ \$(oc get services.serving.knative.dev upgrade-probe \ + -n serving-tests -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') \ + != True ]]" || return $? - # Might not work in OpenShift CI but we want it here so that we can consume this script later and re-use - if [[ $UPGRADE_CLUSTER == true ]]; then - # End the prober test now before we start cluster upgrade, up until now we should have zero failed requests - end_prober_test ${PROBER_PID} || return $? - - if [[ -n "$UPGRADE_OCP_IMAGE" ]]; then - oc adm upgrade --to-image="${UPGRADE_OCP_IMAGE}" --force=true --allow-explicit-upgrade - timeout 7200 '[[ $(oc get clusterversion -o=jsonpath="{.items[0].status.history[?(@.image==\"${UPGRADE_OCP_IMAGE}\")].state}") != Completed ]]' || return 1 - else - latest_cluster_version=$(oc adm upgrade | sed -ne '/VERSION/,$ p' | grep -v VERSION | awk '{print $1}' | sort -r | head -n 1) - [[ $latest_cluster_version != "" ]] || return 1 - oc adm upgrade --to-latest=true --force=true - timeout 7200 '[[ $(oc get clusterversion -o=jsonpath="{.items[0].status.history[?(@.version==\"${latest_cluster_version}\")].state}") != Completed ]]' || return 1 - fi - - logger.info "New cluster version\n: $(oc get clusterversion)" - fi + logger.success 'Serving prober is ready' +} + +function check_serving_upgraded { + local latest_serving_version + latest_serving_version="${1:?Pass a target serving version as arg[1]}" + + logger.debug 'Check KnativeServing has the latest version with Ready status' + timeout 300 "[[ ! ( \$(oc get knativeserving.operator.knative.dev \ + knative-serving -n ${SERVING_NAMESPACE} -o=jsonpath='{.status.version}') \ + == ${latest_serving_version} && \$(oc get knativeserving.operator.knative.dev \ + knative-serving -n ${SERVING_NAMESPACE} \ + -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" \ + || return $? +} + +function end_serving_prober { + local prober_pid + prober_pid="${1:?Pass a prober pid as arg[1]}" - # Wait for all services to become ready again. Exclude the upgrade-probe as that'll be removed by the prober test above. - for kservice in $(oc get ksvc -n serving-tests --no-headers -o name | grep -v "upgrade-probe"); do - timeout 900 '[[ $(oc get $kservice -n serving-tests -o=jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}") != True ]]' || return 1 + end_prober 'Serving' "${prober_pid}" || return $? +} + +function wait_for_serving_test_services_settle { + # Wait for all services to become ready again. Exclude the upgrade-probe as + # that'll be removed by the prober test above. + for kservice in $(oc get ksvc -n serving-tests --no-headers -o name | grep -v 'upgrade-probe'); do + timeout 900 "[[ \$(oc get ${kservice} -n serving-tests -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]" || return $? done # Give time to settle things down sleep 30 +} + +function run_serving_postupgrade_test { + logger.info 'Running Serving post upgrade tests' + + local image_template - logger.info "Running postupgrade tests" - go_test_e2e -tags=postupgrade -timeout=20m ./test/upgrade \ + cd "${KNATIVE_SERVING_HOME}" || return $? + + image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}" + + go_test_e2e -tags=postupgrade \ + -timeout=20m ./test/upgrade \ --imagetemplate "$image_template" \ --kubeconfig "$KUBECONFIG" \ - --resolvabledomain || return 1 + --resolvabledomain || return $? - oc delete --ignore-not-found=true ksvc pizzaplanet-upgrade-service scale-to-zero-upgrade-service upgrade-probe -n serving-tests + logger.success 'Serving post upgrade tests passed' +} - return 0 - ) +function cleanup_serving_test_services { + oc delete --ignore-not-found=true ksvc \ + pizzaplanet-upgrade-service \ + scale-to-zero-upgrade-service \ + upgrade-probe -n serving-tests \ + || return $? } diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh index bc039bb0ee..271b5ec1fe 100755 --- a/test/upstream-e2e-tests.sh +++ b/test/upstream-e2e-tests.sh @@ -11,25 +11,21 @@ if [ -n "$OPENSHIFT_CI" ]; then fi debugging.setup -create_namespaces || exit $? - failed=0 teardown_serverless || failed=1 -(( !failed )) && install_catalogsource || failed=2 +(( !failed )) && create_namespaces || failed=2 +(( !failed )) && install_catalogsource || failed=3 (( !failed )) && logger.success '🚀 Cluster prepared for testing.' # Run upgrade tests if [[ $TEST_KNATIVE_UPGRADE == true ]]; then - # TODO(markusthoemmes): Remove after 1.11 is cut. - (( !failed )) && oc create namespace "${SERVING_NAMESPACE}" - - (( !failed )) && install_serverless_previous || failed=3 - (( !failed )) && run_knative_serving_rolling_upgrade_tests || failed=4 - (( !failed )) && trigger_gc_and_print_knative || failed=5 + (( !failed )) && install_serverless_previous || failed=4 + (( !failed )) && run_rolling_upgrade_tests "${UPGRADE_TEST_SCOPE:-serving,eventing}" || failed=5 + (( !failed )) && trigger_gc_and_print_knative || failed=6 # Call teardown only if E2E tests follow. if [[ $TEST_KNATIVE_E2E == true ]]; then - (( !failed )) && teardown_serverless || failed=6 + (( !failed )) && teardown_serverless || failed=7 fi fi