From 0047b93111ba46a1a099caf0928b8f417b60423c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Thu, 8 Oct 2020 18:31:19 +0200
Subject: [PATCH 01/13] Splitting upgrade test to accommodate Eventing

---
 hack/lib/catalogsource.bash |  16 ++--
 hack/lib/common.bash        |  23 +++++
 hack/lib/serverless.bash    |  49 +++++-----
 hack/lib/vars.bash          |   8 +-
 test/eventing.bash          |  69 ++++++++++++++-
 test/lib.bash               | 131 +++++++++++++++++++++++++--
 test/serving.bash           | 172 ++++++++++++++++++++----------------
 test/upstream-e2e-tests.sh  |   2 +-
 8 files changed, 352 insertions(+), 118 deletions(-)

diff --git a/hack/lib/catalogsource.bash b/hack/lib/catalogsource.bash
index 43402b0f8a..5330eea4e9 100644
--- a/hack/lib/catalogsource.bash
+++ b/hack/lib/catalogsource.bash
@@ -12,7 +12,9 @@ function ensure_catalogsource_installed {
 function install_catalogsource {
   logger.info "Installing CatalogSource"
 
-  local rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
+  local rootdir
+
+  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
 
   # Add a user that is allowed to pull images from the registry.
   pull_user="puller"
@@ -33,9 +35,9 @@ function install_catalogsource {
     sed -i "s,image: .*openshift-serverless-.*:knative-openshift-ingress,image: ${DOCKER_REPO_OVERRIDE}/knative-openshift-ingress," "$csv"
   fi
 
-  cat "$csv"
+  [ -n "$OPENSHIFT_CI" ] && cat "$csv"
 
-  # Build the bundle image in the cluster-internal registry.
+  logger.info 'Build the bundle image in the cluster-internal registry.'
   oc -n "$OLM_NAMESPACE" new-build --binary --strategy=docker --name serverless-bundle
   oc -n "$OLM_NAMESPACE" start-build serverless-bundle --from-dir olm-catalog/serverless-operator -F
 
@@ -45,7 +47,7 @@ function install_catalogsource {
   # HACK: Allow to run the index pod as root so it has necessary access.
   oc -n "$OLM_NAMESPACE" adm policy add-scc-to-user anyuid -z default
 
-  # Install the index deployment.
+  logger.info 'Install the index deployment.'
   # This image was built using the Dockerfile at 'olm-catalog/serverless-operator/index.Dockerfile'.
   cat <<EOF | oc apply -n "$OLM_NAMESPACE" -f - || return $? 
 apiVersion: apps/v1
@@ -82,11 +84,11 @@ spec:
           /bin/opm registry serve -d index.db -p 50051
 EOF
 
-  # Wait for the index pod to be up to avoid inconsistencies with the catalog source.
+  logger.info 'Wait for the index pod to be up to avoid inconsistencies with the catalog source.'
   wait_until_pods_running "$OLM_NAMESPACE"
   indexip="$(oc -n "$OLM_NAMESPACE" get pods -l app=serverless-index -ojsonpath='{.items[0].status.podIP}')"
 
-  # Install the catalogsource.
+  logger.info 'Install the catalogsource.'
   cat <<EOF | oc apply -n "$OLM_NAMESPACE" -f - || return $?
 apiVersion: operators.coreos.com/v1alpha1
 kind: CatalogSource
@@ -134,6 +136,6 @@ function add_user {
 
   logger.info 'Generate kubeconfig'
   cp "${KUBECONFIG}" "$name.kubeconfig"
-  occmd="bash -c '! oc login --config=$name.kubeconfig --username=$name --password=$pass > /dev/null'"
+  occmd="bash -c '! oc login --kubeconfig=${name}.kubeconfig --username=${name} --password=${pass} > /dev/null'"
   timeout 900 "${occmd}" || return 1
 }
diff --git a/hack/lib/common.bash b/hack/lib/common.bash
index 17cba090a2..75345a8a3f 100644
--- a/hack/lib/common.bash
+++ b/hack/lib/common.bash
@@ -56,3 +56,26 @@ function wait_until_hostname_resolves() {
   logger.error "Timeout waiting for hostname $1 to resolve via DNS"
   return 1
 }
+
+function wait_for_file {
+  local file timeout waits
+  file="${1:?Pass a filepath as arg[1]}"
+  waits="${2:-300}"
+  timeout=$waits
+
+  logger.debug "Waiting for existence of file: ${file}"
+
+  while [ ! -f "${file}" ]; do
+    # When the timeout is equal to zero, show an error and leave the loop.
+    if [ "${timeout}" == 0 ]; then
+      logger.error "Timeout (${waits}s) while waiting for the file ${file}."
+      return 78
+    fi
+
+    sleep 1
+
+    # Decrease the timeout of one
+    ((timeout--))
+  done
+  return 0
+}
diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash
index 420beca33c..73503ff184 100644
--- a/hack/lib/serverless.bash
+++ b/hack/lib/serverless.bash
@@ -22,8 +22,7 @@ function ensure_serverless_installed {
 }
 
 function install_serverless_previous {
-  local rootdir
-  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
+  logger.info "Installing previous version of Serverless..."
 
   # Remove installplan from previous installations, leaving this would make the operator
   # upgrade to the latest version immediately
@@ -31,34 +30,40 @@ function install_serverless_previous {
 
   deploy_serverless_operator "$PREVIOUS_CSV"  || return $?
   deploy_knativeserving_cr || return $?
-}
-
-function remove_installplan {
-  local install_plan
-  install_plan=$(find_install_plan $1)
-  if [[ -n $install_plan ]]; then
-    oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}"
-  fi
+  deploy_knativeeventing_cr || return $?
+  logger.success "Previous version of Serverless is installed: $PREVIOUS_CSV"
 }
 
 function install_serverless_latest {
+  logger.info "Installing latest version of Serverless..."
   deploy_serverless_operator_latest || return $?
   deploy_knativeserving_cr || return $?
   deploy_knativeeventing_cr || return $?
+  logger.success "Latest version of Serverless is installed: $CURRENT_CSV"
+}
+
+function remove_installplan {
+  local install_plan csv
+  csv="${1:?Pass a CSV as arg[1]}"
+  logger.info "Removing installplan for $csv"
+  install_plan=$(find_install_plan "$csv")
+  if [[ -n $install_plan ]]; then
+    oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}"
+  else
+    logger.debug "No install plan for $csv"
+  fi
 }
 
 function deploy_serverless_operator_latest {
-  local rootdir
-  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
   deploy_serverless_operator "$CURRENT_CSV"
 }
 
 function deploy_serverless_operator {
-  local csv
-  csv="$1"
-  logger.info "Install the Serverless Operator ${csv}"
-
-  cat <<EOF | oc apply -f - || return $?
+  local csv tmpfile
+  csv="${1:?Pass as CSV as arg[1]}"
+  logger.info "Install the Serverless Operator: ${csv}"
+  tmpfile=$(mktemp /tmp/subscription.XXXXXX.yaml)
+  cat > "$tmpfile" <<EOF
 apiVersion: operators.coreos.com/v1alpha1
 kind: Subscription
 metadata:
@@ -72,6 +77,8 @@ spec:
   installPlanApproval: Manual
   startingCSV: "${csv}"
 EOF
+  [ -n "$OPENSHIFT_CI" ] && cat "$tmpfile"
+  oc apply -f "$tmpfile" || return $?
 
   # Approve the initial installplan automatically
   approve_csv "$csv" "$OLM_CHANNEL" || return 5
@@ -79,16 +86,16 @@ EOF
 
 function approve_csv {
   local csv_version install_plan channel
-  csv_version=$1
-  channel=$2
+  csv_version=${1:?Pass a CSV as arg[1]}
+  channel=${2:?Pass channel as arg[2]}
 
-  # Ensure channel and source is set properly
+  logger.info 'Ensure channel and source is set properly'
   oc patch subscriptions.operators.coreos.com "$OPERATOR" -n "${OPERATORS_NAMESPACE}" \
     --type 'merge' \
     --patch '{"spec": {"channel": "'"${channel}"'", "source": "'"${OLM_SOURCE}"'"}}' \
     || return $?
 
-  # Wait for the installplan to be available
+  logger.info 'Wait for the installplan to be available'
   timeout 900 "[[ -z \$(find_install_plan $csv_version) ]]" || return 1
 
   install_plan=$(find_install_plan $csv_version)
diff --git a/hack/lib/vars.bash b/hack/lib/vars.bash
index 7444963e2f..066c60dfca 100644
--- a/hack/lib/vars.bash
+++ b/hack/lib/vars.bash
@@ -18,8 +18,9 @@ export KNATIVE_EVENTING_VERSION="${KNATIVE_EVENTING_VERSION:-v0.16.0}"
 # Make sure yq is on PATH.
 yq > /dev/null || exit 127
 csv_file="$(dirname "${BASH_SOURCE[0]}")/../../olm-catalog/serverless-operator/manifests/serverless-operator.clusterserviceversion.yaml"
-export CURRENT_CSV="$(yq r "$csv_file" metadata.name)"
-export PREVIOUS_CSV="$(yq r "$csv_file" spec.replaces)"
+CURRENT_CSV="$(yq r "$csv_file" metadata.name)"
+PREVIOUS_CSV="$(yq r "$csv_file" spec.replaces)"
+export CURRENT_CSV PREVIOUS_CSV
 
 # Directories below are filled with source code by ci-operator
 export KNATIVE_SERVING_HOME="${GOPATH}/src/knative.dev/serving"
@@ -59,7 +60,8 @@ function latest_channel_from_metadata {
   echo "$channels" | awk -F"," '{ print $NF }'
 }
 
-export OLM_CHANNEL="${OLM_CHANNEL:-"$(latest_channel_from_metadata)"}"
+OLM_CHANNEL="${OLM_CHANNEL:-"$(latest_channel_from_metadata)"}"
+export OLM_CHANNEL
 # Change this when upgrades need switching to a different channel
 export OLM_UPGRADE_CHANNEL="${OLM_UPGRADE_CHANNEL:-"$OLM_CHANNEL"}"
 export OLM_SOURCE="${OLM_SOURCE:-"$OPERATOR"}"
diff --git a/test/eventing.bash b/test/eventing.bash
index 9df32e769d..5b399d8f6d 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -1,15 +1,19 @@
 #!/usr/bin/env bash
 
+readonly EVENTING_READY_FILE="/tmp/eventing-prober-ready"
+readonly EVENTING_PROBER_FILE="/tmp/eventing-prober-signal"
+
 function upstream_knative_eventing_e2e {
   (
   logger.info 'Running eventing tests'
 
   local failed=0
 
-  TEST_IMAGE_TEMPLATE="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+  export TEST_IMAGE_TEMPLATE="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
 
-  cd "$KNATIVE_EVENTING_HOME" || return $?
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
 
+  # shellcheck disable=SC1090
   source "${KNATIVE_EVENTING_HOME}/openshift/e2e-common.sh"
 
   # run_e2e_tests defined in knative-eventing
@@ -20,3 +24,64 @@ function upstream_knative_eventing_e2e {
   return $failed
   )
 }
+
+function actual_eventing_version {
+  oc get knativeeventing.operator.knative.dev \
+    knative-eventing -n "${EVENTING_NAMESPACE}" -o=jsonpath="{.status.version}"
+}
+
+function run_eventing_preupgrade_test {
+  logger.info 'Running Eventing pre upgrade tests'
+
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
+
+  go_test_e2e -tags=preupgrade -timeout=10m ./test/upgrade || return $?
+
+  logger.success 'Eventing pre upgrade tests passed'
+}
+
+function start_eventing_prober {
+  local eventing_prober_pid result_file
+  result_file="${1:?Pass a result file as arg[1]}"
+
+  logger.info 'Starting Eventing prober'
+
+  rm -fv "${EVENTING_PROBER_FILE}" "${EVENTING_READY_FILE}"
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
+
+  go_test_e2e -tags=probe \
+    -timeout=30m \
+    ./test/upgrade \
+    --pipefile="${EVENTING_PROBER_FILE}" \
+    --readyfile="${EVENTING_READY_FILE}" &
+  eventing_prober_pid=$!
+
+  logger.debug "Eventing prober PID is ${eventing_prober_pid}"
+
+  echo ${eventing_prober_pid} > "${result_file}"
+}
+
+function wait_for_eventing_prober_ready {
+  wait_for_file "${EVENTING_READY_FILE}" || return $?
+
+  logger.success 'Eventing prober is ready'
+}
+
+function end_eventing_prober {
+  local prober_pid
+  prober_pid="${1:?Pass a prober pid as arg[1]}"
+
+  end_prober_test 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}" || return $?
+}
+
+function run_eventing_postupgrade_test {
+  logger.info 'Running Eventing post upgrade tests'
+
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
+
+  go_test_e2e -tags=postupgrade -timeout=10m ./test/upgrade || return $?
+
+  logger.success 'Eventing post upgrade tests passed'
+}
+
+
diff --git a/test/lib.bash b/test/lib.bash
index d228e7a899..dd7b22c012 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# == Overrides & test releated
+# == Overrides & test related
 
 # shellcheck disable=SC1091,SC1090
 source "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")/hack/lib/__sources__.bash"
@@ -115,12 +115,131 @@ function downstream_eventing_e2e_tests {
   return $failed
 }
 
+# == Upgrade testing
+
+function run_rolling_upgrade_tests {
+  logger.info "Running rolling upgrade tests"
+  (
+  local latest_cluster_version latest_serving_version latest_eventing_version \
+    rootdir scope serving_in_scope eventing_in_scope serving_prober_pid \
+    eventing_prober_pid prev_serving_version prev_eventing_version \
+    ocp_target_version
+
+  scope="${1:?Provide an upgrade scope as arg[1]}"
+  serving_in_scope="$(echo "${scope}" | grep -vq serving ; echo "$?")"
+  eventing_in_scope="$(echo "${scope}" | grep -vq eventing ; echo "$?")"
+
+  prev_serving_version="$(actual_serving_version)"
+  prev_eventing_version="$(actual_eventing_version)"
+
+  # Save the rootdir before changing dir
+  rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")"
+
+  (( serving_in_scope )) && prepare_knative_serving_tests || return $?
+
+  logger.info 'Testing with pre upgrade tests'
+
+  (( serving_in_scope )) && run_serving_preupgrade_test || return $?
+  (( eventing_in_scope )) && run_eventing_preupgrade_test || return $?
+
+  logger.info 'Starting prober tests'
+
+  if (( serving_in_scope )); then
+    start_serving_prober "${prev_serving_version}" /tmp/prober-pid || return $?
+    serving_prober_pid=$(cat /tmp/prober-pid)
+  fi
+  if (( eventing_in_scope )); then
+    start_eventing_prober /tmp/prober-pid || return $?
+    eventing_prober_pid=$(cat /tmp/prober-pid)
+  fi
+
+  (( serving_in_scope )) && wait_for_serving_prober_ready || return $?
+  (( eventing_in_scope )) && wait_for_eventing_prober_ready || return $?
+
+  if [[ $UPGRADE_SERVERLESS == true ]]; then
+    latest_serving_version="${KNATIVE_SERVING_VERSION/v/}"
+    latest_eventing_version="${KNATIVE_EVENTING_VERSION/v/}"
+
+    logger.info "Updating Serverless to ${CURRENT_CSV}"
+    logger.debug "Serving version: ${prev_serving_version} -> ${latest_serving_version}"
+    logger.debug "Eventing version: ${prev_eventing_version} -> ${latest_eventing_version}"
+
+    approve_csv "$CURRENT_CSV" "$OLM_UPGRADE_CHANNEL" || return $?
+    (( serving_in_scope )) && check_serving_upgraded "${latest_serving_version}" || return $?
+    (( eventing_in_scope )) && check_eventing_upgraded "${latest_eventing_version}" || return $?
+  fi
+
+  # Might not work in OpenShift CI but we want it here so that we can consume
+  # this script later and re-use
+  if [[ $UPGRADE_CLUSTER == true ]]; then
+    # End the prober test now before we start cluster upgrade, up until now we
+    # should have zero failed requests. Cluster upgrade will fail probers as
+    # stuff is moved around.
+    (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}" || return $?
+    (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}" || return $?
+
+    upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}" || return $?
+  fi
+
+  (( serving_in_scope )) && wait_for_serving_test_services_settle || return $?
+
+  logger.info "Running postupgrade tests"
+
+  (( serving_in_scope )) && run_serving_postupgrade_test || return $?
+  (( eventing_in_scope )) && run_eventing_postupgrade_test || return $?
+
+  (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}" || return $?
+  (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}" || return $?
+
+  cleanup_serving_test_servinces || return $?
+
+  cd "$rootdir" || return $?
+  return 0
+  )
+}
+
 function end_prober_test {
-  local PROBER_PID=$1
-  echo "done" > /tmp/prober-signal
-  logger.info "Waiting for prober test to finish"
-  wait "${PROBER_PID}"
-  return $?
+  local prober_pid prober_signal retcode title
+  title=${1:?Pass a title as arg[1]}
+  prober_pid=${2:?Pass a pid as a arg[2]}
+  prober_signal=${3:-/tmp/prober-signal}
+
+  if kill -0 "${prober_pid}" 2>/dev/null; then
+    logger.debug "${title} prober of PID ${prober_pid} isn't running..."
+    return 0
+  fi
+
+  echo 'done' > "${prober_signal}"
+  logger.info "Waiting for ${title} prober test to finish"
+  wait "${prober_pid}"
+  retcode=$?
+  if ! (( retcode )); then
+    logger.success "${title} prober passed"
+  else
+    logger.error "${title} prober failed"
+  fi
+  return $retcode
+}
+
+function upgrade_ocp_cluster {
+  local ocp_target_version upgrade_ocp_image latest_cluster_version
+  upgrade_ocp_image="${1:-}"
+
+  if [[ -n "$upgrade_ocp_image" ]]; then
+    ocp_target_version="$upgrade_ocp_image"
+    oc adm upgrade --to-image="${UPGRADE_OCP_IMAGE}" \
+      --force=true --allow-explicit-upgrade
+  else
+    latest_cluster_version=$(oc adm upgrade | sed -ne '/VERSION/,$ p' \
+      | grep -v VERSION | awk '{print $1}' | sort -r | head -n 1)
+    [[ $latest_cluster_version != "" ]] || return 1
+    ocp_target_version="$latest_cluster_version"
+    oc adm upgrade --to-latest=true --force=true
+  fi
+  timeout 7200 "[[ \$(oc get clusterversion version -o jsonpath='{.status.history[?(@.image==\"${ocp_target_version}\")].state}') != Completed ]]" || return 1
+
+  logger.success "New cluster version: $(oc get clusterversion \
+    version -o jsonpath='{.status.desired.version}')"
 }
 
 function teardown {
diff --git a/test/serving.bash b/test/serving.bash
index 34624e7708..63077d410e 100644
--- a/test/serving.bash
+++ b/test/serving.bash
@@ -12,6 +12,10 @@ function wait_for_knative_serving_ingress_ns_deleted {
 }
 
 function prepare_knative_serving_tests {
+  logger.debug 'Preparing Serving tests'
+
+  cd "$KNATIVE_SERVING_HOME" || return $?
+
   # Create test resources (namespaces, configMaps, secrets)
   oc apply -f test/config
   oc adm policy add-scc-to-user privileged -z default -n serving-tests
@@ -30,8 +34,6 @@ function prepare_knative_serving_tests {
 function upstream_knative_serving_e2e_and_conformance_tests {
   logger.info "Running Serving E2E and conformance tests"
   (
-  cd "$KNATIVE_SERVING_HOME" || return $?
-
   prepare_knative_serving_tests || return $?
 
   local failed=0
@@ -77,107 +79,121 @@ function upstream_knative_serving_e2e_and_conformance_tests {
   )
 }
 
-function run_knative_serving_rolling_upgrade_tests {
-  logger.info "Running Serving rolling upgrade tests"
-  (
-  local failed upgrade_to latest_cluster_version cluster_version prev_serving_version latest_serving_version
+function actual_serving_version {
+  oc get knativeserving.operator.knative.dev \
+    knative-serving -n "${SERVING_NAMESPACE}" -o=jsonpath="{.status.version}"
+}
 
-  # Save the rootdir before changing dir
-  rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")"
+function run_serving_preupgrade_test {
+  logger.info 'Running Serving pre upgrade tests'
 
-  cd "$KNATIVE_SERVING_HOME" || return $?
+  local image_template
 
-  prepare_knative_serving_tests || return $?
+  cd "${KNATIVE_SERVING_HOME}" || return $?
 
-  failed=0
   image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}"
-  PROBE_FRACTION=1.0
-  prev_serving_version=$(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.version}")
-
-  if [[ ${prev_serving_version} < "0.14.0" ]]; then
-    PROBE_FRACTION=0.95
-  fi
-  logger.info "Target success fraction is $PROBE_FRACTION"
 
   go_test_e2e -tags=preupgrade -timeout=20m ./test/upgrade \
     --imagetemplate "$image_template" \
     --kubeconfig "$KUBECONFIG" \
-    --resolvabledomain || return 1
+    --resolvabledomain || return $?
+
+  logger.success 'Serving pre upgrade tests passed'
+}
+
+function start_serving_prober {
+  local image_template prev_serving_version probe_fraction serving_prober_pid \
+    result_file
+  prev_serving_version="${1:?Pass a previous Serving version as arg[1]}"
+  result_file="${2:?Pass a result file as arg[2]}"
+
+  logger.info 'Starting Serving prober'
 
-  logger.info "Starting prober test"
+  rm -fv /tmp/prober-signal
+  cd "${KNATIVE_SERVING_HOME}" || return $?
+
+  probe_fraction=1.0
+  if [[ ${prev_serving_version} < "0.14.0" ]]; then
+    probe_fraction=0.95
+  fi
+  logger.info "Target success fraction for Serving is ${probe_fraction}"
+
+  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}"
 
-  rm -f /tmp/prober-signal
-  go_test_e2e -tags=probe -timeout=20m ./test/upgrade \
-    -probe.success_fraction=$PROBE_FRACTION \
+  go_test_e2e -tags=probe \
+    -timeout=30m \
+    ./test/upgrade \
+    -probe.success_fraction=${probe_fraction} \
     --imagetemplate "$image_template" \
     --kubeconfig "$KUBECONFIG" \
     --resolvabledomain &
+  serving_prober_pid=$!
 
+  logger.debug "Serving prober PID is ${serving_prober_pid}"
+
+  echo ${serving_prober_pid} > "${result_file}"
+}
+
+function wait_for_serving_prober_ready {
   # Wait for the upgrade-probe kservice to be ready before proceeding
-  timeout 900 '[[ $(oc get services.serving.knative.dev upgrade-probe -n serving-tests -o=jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}") != True ]]' || return 1
-
-  PROBER_PID=$!
-
-  if [[ $UPGRADE_SERVERLESS == true ]]; then
-    latest_serving_version=$(echo $KNATIVE_SERVING_VERSION | sed "s/v//")
-
-    logger.info "updating serving version from ${prev_serving_version} to ${latest_serving_version}"
-
-    # Get latest CSV from the given channel
-    upgrade_to="$CURRENT_CSV"
-
-    cluster_version=$(oc get clusterversion -o=jsonpath="{.items[0].status.history[?(@.state==\"Completed\")].version}")
-    if [[ "$cluster_version" = 4.1.* || "${HOSTNAME}" = *ocp-41* || \
-          "$cluster_version" = 4.2.* || "${HOSTNAME}" = *ocp-42* ]]; then
-      if approve_csv "$upgrade_to" "$OLM_UPGRADE_CHANNEL" ; then # Upgrade should fail on OCP 4.1, 4.2
-        return 1
-      fi
-      # Check we got RequirementsNotMet error
-      [[ $(oc get ClusterServiceVersion $upgrade_to -n $OPERATORS_NAMESPACE -o=jsonpath="{.status.requirementStatus[?(@.name==\"$upgrade_to\")].message}") =~ "requirement not met: minKubeVersion" ]] || return 1
-      # Check KnativeServing still has the old version
-      [[ $(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.version}") == "$prev_serving_version" ]] || return 1
-    else
-      approve_csv "$upgrade_to" "$OLM_UPGRADE_CHANNEL" || return 1
-      # Check KnativeServing has the latest version with Ready status
-      timeout 300 '[[ ! ( $(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.version}") == $latest_serving_version && $(oc get knativeserving.operator.knative.dev knative-serving -n $SERVING_NAMESPACE -o=jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}") == True ) ]]' || return 1
-    fi
-    end_prober_test ${PROBER_PID} || return $?
-  fi
+  timeout 900 "[[ \$(oc get services.serving.knative.dev upgrade-probe \
+    -n serving-tests -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') \
+    != True ]]" || return $?
 
-  # Might not work in OpenShift CI but we want it here so that we can consume this script later and re-use
-  if [[ $UPGRADE_CLUSTER == true ]]; then
-    # End the prober test now before we start cluster upgrade, up until now we should have zero failed requests
-    end_prober_test ${PROBER_PID} || return $?
-
-    if [[ -n "$UPGRADE_OCP_IMAGE" ]]; then
-      oc adm upgrade --to-image="${UPGRADE_OCP_IMAGE}" --force=true --allow-explicit-upgrade
-      timeout 7200 '[[ $(oc get clusterversion -o=jsonpath="{.items[0].status.history[?(@.image==\"${UPGRADE_OCP_IMAGE}\")].state}") != Completed ]]' || return 1
-    else
-      latest_cluster_version=$(oc adm upgrade | sed -ne '/VERSION/,$ p' | grep -v VERSION | awk '{print $1}' | sort -r | head -n 1)
-      [[ $latest_cluster_version != "" ]] || return 1
-      oc adm upgrade --to-latest=true --force=true
-      timeout 7200 '[[ $(oc get clusterversion -o=jsonpath="{.items[0].status.history[?(@.version==\"${latest_cluster_version}\")].state}") != Completed ]]' || return 1
-    fi
-
-    logger.info "New cluster version\n: $(oc get clusterversion)"
-  fi
+  logger.success 'Serving prober is ready'
+}
+
+function check_serving_upgraded {
+  local latest_serving_version
+  latest_serving_version="${1:?Pass a target serving version as arg[1]}"
+
+  logger.debug 'Check KnativeServing has the latest version with Ready status'
+  timeout 300 "[[ ! ( \$(oc get knativeserving.operator.knative.dev \
+    knative-serving -n ${SERVING_NAMESPACE} -o=jsonpath='{.status.version}') \
+    == ${latest_serving_version} && \$(oc get knativeserving.operator.knative.dev \
+    knative-serving -n ${SERVING_NAMESPACE} \
+    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" || return 1
+}
+
+function end_serving_prober {
+  local prober_pid
+  prober_pid="${1:?Pass a prober pid as arg[1]}"
 
-  # Wait for all services to become ready again. Exclude the upgrade-probe as that'll be removed by the prober test above.
-  for kservice in $(oc get ksvc -n serving-tests --no-headers -o name | grep -v "upgrade-probe"); do
-    timeout 900 '[[ $(oc get $kservice -n serving-tests -o=jsonpath="{.status.conditions[?(@.type==\"Ready\")].status}") != True ]]' || return 1
+  end_prober_test 'Serving' "${prober_pid}" || return $?
+}
+
+function wait_for_serving_test_services_settle {
+  # Wait for all services to become ready again. Exclude the upgrade-probe as
+  # that'll be removed by the prober test above.
+  for kservice in $(oc get ksvc -n serving-tests --no-headers -o name | grep -v 'upgrade-probe'); do
+    timeout 900 "[[ \$(oc get ${kservice} -n serving-tests -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]" || return 1
   done
 
   # Give time to settle things down
   sleep 30
+}
+
+function run_serving_postupgrade_test {
+  logger.info 'Running Serving post upgrade tests'
+
+  local image_template
 
-  logger.info "Running postupgrade tests"
-  go_test_e2e -tags=postupgrade -timeout=20m ./test/upgrade \
+  cd "${KNATIVE_SERVING_HOME}" || return $?
+
+  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}"
+
+  go_test_e2e -tags=postupgrade \
+    -timeout=20m ./test/upgrade \
     --imagetemplate "$image_template" \
     --kubeconfig "$KUBECONFIG" \
-    --resolvabledomain || return 1
+    --resolvabledomain || return $?
 
-  oc delete --ignore-not-found=true ksvc pizzaplanet-upgrade-service scale-to-zero-upgrade-service upgrade-probe -n serving-tests
+  logger.success 'Serving post upgrade tests passed'
+}
 
-  return 0
-  )
+function cleanup_serving_test_servinces {
+  oc delete --ignore-not-found=true ksvc \
+    pizzaplanet-upgrade-service \
+    scale-to-zero-upgrade-service \
+    upgrade-probe -n serving-tests
 }
diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh
index 8db341bab4..72874157c0 100755
--- a/test/upstream-e2e-tests.sh
+++ b/test/upstream-e2e-tests.sh
@@ -22,7 +22,7 @@ teardown_serverless || failed=1
 # Run upgrade tests
 if [[ $TEST_KNATIVE_UPGRADE == true ]]; then
   (( !failed )) && install_serverless_previous || failed=3
-  (( !failed )) && run_knative_serving_rolling_upgrade_tests || failed=4
+  (( !failed )) && run_rolling_upgrade_tests "${UPGRADE_TEST_SCOPE:-serving}" || failed=4
   (( !failed )) && trigger_gc_and_print_knative || failed=5
   # Call teardown only if E2E tests follow.
   if [[ $TEST_KNATIVE_E2E == true ]]; then

From 5047f4c3dc1fbda397ec1d95fef2dbee7e815fbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Thu, 8 Oct 2020 19:48:58 +0200
Subject: [PATCH 02/13] Add an image template

---
 test/eventing.bash | 25 +++++++++++++++++++------
 test/lib.bash      | 38 +++++++++++++++++++-------------------
 test/serving.bash  | 10 +++++-----
 3 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/test/eventing.bash b/test/eventing.bash
index 5b399d8f6d..5227c5c628 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -35,13 +35,18 @@ function run_eventing_preupgrade_test {
 
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
-  go_test_e2e -tags=preupgrade -timeout=10m ./test/upgrade || return $?
+  local image_template
+  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+
+  go_test_e2e -tags=preupgrade \
+    -timeout=10m ./test/upgrade \
+    --imagetemplate="${image_template}"
 
   logger.success 'Eventing pre upgrade tests passed'
 }
 
 function start_eventing_prober {
-  local eventing_prober_pid result_file
+  local eventing_prober_pid result_file image_template
   result_file="${1:?Pass a result file as arg[1]}"
 
   logger.info 'Starting Eventing prober'
@@ -49,11 +54,14 @@ function start_eventing_prober {
   rm -fv "${EVENTING_PROBER_FILE}" "${EVENTING_READY_FILE}"
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
+  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+
   go_test_e2e -tags=probe \
     -timeout=30m \
     ./test/upgrade \
     --pipefile="${EVENTING_PROBER_FILE}" \
-    --readyfile="${EVENTING_READY_FILE}" &
+    --readyfile="${EVENTING_READY_FILE}" \
+    --imagetemplate="${image_template}" &
   eventing_prober_pid=$!
 
   logger.debug "Eventing prober PID is ${eventing_prober_pid}"
@@ -62,7 +70,7 @@ function start_eventing_prober {
 }
 
 function wait_for_eventing_prober_ready {
-  wait_for_file "${EVENTING_READY_FILE}" || return $?
+  wait_for_file "${EVENTING_READY_FILE}"
 
   logger.success 'Eventing prober is ready'
 }
@@ -71,15 +79,20 @@ function end_eventing_prober {
   local prober_pid
   prober_pid="${1:?Pass a prober pid as arg[1]}"
 
-  end_prober_test 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}" || return $?
+  end_prober_test 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}"
 }
 
 function run_eventing_postupgrade_test {
   logger.info 'Running Eventing post upgrade tests'
+  local image_template
 
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
-  go_test_e2e -tags=postupgrade -timeout=10m ./test/upgrade || return $?
+  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+
+  go_test_e2e -tags=postupgrade \
+    -timeout=10m ./test/upgrade \
+    --imagetemplate="${image_template}"
 
   logger.success 'Eventing post upgrade tests passed'
 }
diff --git a/test/lib.bash b/test/lib.bash
index dd7b22c012..00d1d4802e 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -135,26 +135,26 @@ function run_rolling_upgrade_tests {
   # Save the rootdir before changing dir
   rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")"
 
-  (( serving_in_scope )) && prepare_knative_serving_tests || return $?
+  (( serving_in_scope )) && prepare_knative_serving_tests
 
   logger.info 'Testing with pre upgrade tests'
 
-  (( serving_in_scope )) && run_serving_preupgrade_test || return $?
-  (( eventing_in_scope )) && run_eventing_preupgrade_test || return $?
+  (( serving_in_scope )) && run_serving_preupgrade_test
+  (( eventing_in_scope )) && run_eventing_preupgrade_test
 
   logger.info 'Starting prober tests'
 
   if (( serving_in_scope )); then
-    start_serving_prober "${prev_serving_version}" /tmp/prober-pid || return $?
+    start_serving_prober "${prev_serving_version}" /tmp/prober-pid
     serving_prober_pid=$(cat /tmp/prober-pid)
   fi
   if (( eventing_in_scope )); then
-    start_eventing_prober /tmp/prober-pid || return $?
+    start_eventing_prober /tmp/prober-pid
     eventing_prober_pid=$(cat /tmp/prober-pid)
   fi
 
-  (( serving_in_scope )) && wait_for_serving_prober_ready || return $?
-  (( eventing_in_scope )) && wait_for_eventing_prober_ready || return $?
+  (( serving_in_scope )) && wait_for_serving_prober_ready
+  (( eventing_in_scope )) && wait_for_eventing_prober_ready
 
   if [[ $UPGRADE_SERVERLESS == true ]]; then
     latest_serving_version="${KNATIVE_SERVING_VERSION/v/}"
@@ -164,9 +164,9 @@ function run_rolling_upgrade_tests {
     logger.debug "Serving version: ${prev_serving_version} -> ${latest_serving_version}"
     logger.debug "Eventing version: ${prev_eventing_version} -> ${latest_eventing_version}"
 
-    approve_csv "$CURRENT_CSV" "$OLM_UPGRADE_CHANNEL" || return $?
-    (( serving_in_scope )) && check_serving_upgraded "${latest_serving_version}" || return $?
-    (( eventing_in_scope )) && check_eventing_upgraded "${latest_eventing_version}" || return $?
+    approve_csv "$CURRENT_CSV" "$OLM_UPGRADE_CHANNEL"
+    (( serving_in_scope )) && check_serving_upgraded "${latest_serving_version}"
+    (( eventing_in_scope )) && check_eventing_upgraded "${latest_eventing_version}"
   fi
 
   # Might not work in OpenShift CI but we want it here so that we can consume
@@ -175,23 +175,23 @@ function run_rolling_upgrade_tests {
     # End the prober test now before we start cluster upgrade, up until now we
     # should have zero failed requests. Cluster upgrade will fail probers as
     # stuff is moved around.
-    (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}" || return $?
-    (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}" || return $?
+    (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}"
+    (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}"
 
-    upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}" || return $?
+    upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}"
   fi
 
-  (( serving_in_scope )) && wait_for_serving_test_services_settle || return $?
+  (( serving_in_scope )) && wait_for_serving_test_services_settle
 
   logger.info "Running postupgrade tests"
 
-  (( serving_in_scope )) && run_serving_postupgrade_test || return $?
-  (( eventing_in_scope )) && run_eventing_postupgrade_test || return $?
+  (( serving_in_scope )) && run_serving_postupgrade_test
+  (( eventing_in_scope )) && run_eventing_postupgrade_test
 
-  (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}" || return $?
-  (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}" || return $?
+  (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}"
+  (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}"
 
-  cleanup_serving_test_servinces || return $?
+  cleanup_serving_test_servinces
 
   cd "$rootdir" || return $?
   return 0
diff --git a/test/serving.bash b/test/serving.bash
index 63077d410e..b6ec47e811 100644
--- a/test/serving.bash
+++ b/test/serving.bash
@@ -96,7 +96,7 @@ function run_serving_preupgrade_test {
   go_test_e2e -tags=preupgrade -timeout=20m ./test/upgrade \
     --imagetemplate "$image_template" \
     --kubeconfig "$KUBECONFIG" \
-    --resolvabledomain || return $?
+    --resolvabledomain
 
   logger.success 'Serving pre upgrade tests passed'
 }
@@ -138,7 +138,7 @@ function wait_for_serving_prober_ready {
   # Wait for the upgrade-probe kservice to be ready before proceeding
   timeout 900 "[[ \$(oc get services.serving.knative.dev upgrade-probe \
     -n serving-tests -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') \
-    != True ]]" || return $?
+    != True ]]"
 
   logger.success 'Serving prober is ready'
 }
@@ -152,21 +152,21 @@ function check_serving_upgraded {
     knative-serving -n ${SERVING_NAMESPACE} -o=jsonpath='{.status.version}') \
     == ${latest_serving_version} && \$(oc get knativeserving.operator.knative.dev \
     knative-serving -n ${SERVING_NAMESPACE} \
-    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" || return 1
+    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]"
 }
 
 function end_serving_prober {
   local prober_pid
   prober_pid="${1:?Pass a prober pid as arg[1]}"
 
-  end_prober_test 'Serving' "${prober_pid}" || return $?
+  end_prober_test 'Serving' "${prober_pid}"
 }
 
 function wait_for_serving_test_services_settle {
   # Wait for all services to become ready again. Exclude the upgrade-probe as
   # that'll be removed by the prober test above.
   for kservice in $(oc get ksvc -n serving-tests --no-headers -o name | grep -v 'upgrade-probe'); do
-    timeout 900 "[[ \$(oc get ${kservice} -n serving-tests -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]" || return 1
+    timeout 900 "[[ \$(oc get ${kservice} -n serving-tests -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]"
   done
 
   # Give time to settle things down

From c772711b94ccdd84646e7911720ec0aaaaae7dc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Fri, 9 Oct 2020 15:10:01 +0200
Subject: [PATCH 03/13] Upgrade tests with Eventing by default

---
 hack/lib/common.bash       | 17 +----------------
 test/lib.bash              |  8 ++++----
 test/serving.bash          |  2 +-
 test/upstream-e2e-tests.sh | 23 +++++++++++------------
 4 files changed, 17 insertions(+), 33 deletions(-)

diff --git a/hack/lib/common.bash b/hack/lib/common.bash
index 75345a8a3f..7a649be808 100644
--- a/hack/lib/common.bash
+++ b/hack/lib/common.bash
@@ -61,21 +61,6 @@ function wait_for_file {
   local file timeout waits
   file="${1:?Pass a filepath as arg[1]}"
   waits="${2:-300}"
-  timeout=$waits
 
-  logger.debug "Waiting for existence of file: ${file}"
-
-  while [ ! -f "${file}" ]; do
-    # When the timeout is equal to zero, show an error and leave the loop.
-    if [ "${timeout}" == 0 ]; then
-      logger.error "Timeout (${waits}s) while waiting for the file ${file}."
-      return 78
-    fi
-
-    sleep 1
-
-    # Decrease the timeout of one
-    ((timeout--))
-  done
-  return 0
+  timeout "${waits}" "[[ ! -f '${file}' ]]" || return $?
 }
diff --git a/test/lib.bash b/test/lib.bash
index 00d1d4802e..5230406899 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -191,7 +191,7 @@ function run_rolling_upgrade_tests {
   (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}"
   (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}"
 
-  cleanup_serving_test_servinces
+  cleanup_serving_test_services
 
   cd "$rootdir" || return $?
   return 0
@@ -206,11 +206,11 @@ function end_prober_test {
 
   if kill -0 "${prober_pid}" 2>/dev/null; then
     logger.debug "${title} prober of PID ${prober_pid} isn't running..."
-    return 0
+  else
+    echo 'done' > "${prober_signal}"
+    logger.info "Waiting for ${title} prober test to finish"
   fi
 
-  echo 'done' > "${prober_signal}"
-  logger.info "Waiting for ${title} prober test to finish"
   wait "${prober_pid}"
   retcode=$?
   if ! (( retcode )); then
diff --git a/test/serving.bash b/test/serving.bash
index b6ec47e811..4b2debe50c 100644
--- a/test/serving.bash
+++ b/test/serving.bash
@@ -191,7 +191,7 @@ function run_serving_postupgrade_test {
   logger.success 'Serving post upgrade tests passed'
 }
 
-function cleanup_serving_test_servinces {
+function cleanup_serving_test_services {
   oc delete --ignore-not-found=true ksvc \
     pizzaplanet-upgrade-service \
     scale-to-zero-upgrade-service \
diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh
index 72874157c0..ebee878aad 100755
--- a/test/upstream-e2e-tests.sh
+++ b/test/upstream-e2e-tests.sh
@@ -11,32 +11,31 @@ if [ -n "$OPENSHIFT_CI" ]; then
 fi
 debugging.setup
 
-create_namespaces || exit $?
-
 failed=0
 
-teardown_serverless || failed=1
-(( !failed )) && install_catalogsource || failed=2
+teardown || failed=1
+(( !failed )) && create_namespaces || failed=2
+(( !failed )) && install_catalogsource || failed=3
 (( !failed )) && logger.success '🚀 Cluster prepared for testing.'
 
 # Run upgrade tests
 if [[ $TEST_KNATIVE_UPGRADE == true ]]; then
-  (( !failed )) && install_serverless_previous || failed=3
-  (( !failed )) && run_rolling_upgrade_tests "${UPGRADE_TEST_SCOPE:-serving}" || failed=4
-  (( !failed )) && trigger_gc_and_print_knative || failed=5
+  (( !failed )) && install_serverless_previous || failed=4
+  (( !failed )) && run_rolling_upgrade_tests "${UPGRADE_TEST_SCOPE:-serving,eventing}" || failed=5
+  (( !failed )) && trigger_gc_and_print_knative || failed=6
   # Call teardown only if E2E tests follow.
   if [[ $TEST_KNATIVE_E2E == true ]]; then
-    (( !failed )) && teardown_serverless || failed=6
+    (( !failed )) && teardown_serverless || failed=7
   fi
 fi
 
 # Run upstream knative serving & eventing tests
 if [[ $TEST_KNATIVE_E2E == true ]]; then
   # Need 6 worker nodes when running upstream.
-  SCALE_UP=6 scale_up_workers || failed=10
-  (( !failed )) && ensure_serverless_installed || failed=7
-  (( !failed )) && upstream_knative_serving_e2e_and_conformance_tests || failed=8
-  (( !failed )) && upstream_knative_eventing_e2e || failed=9
+  SCALE_UP=6 scale_up_workers || failed=8
+  (( !failed )) && ensure_serverless_installed || failed=9
+  (( !failed )) && upstream_knative_serving_e2e_and_conformance_tests || failed=10
+  (( !failed )) && upstream_knative_eventing_e2e || failed=11
 fi
 
 (( failed )) && dump_state

From 06e588b7b126476d2c78f9113dab2682a9808ac2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Fri, 9 Oct 2020 20:22:15 +0200
Subject: [PATCH 04/13] Proper error reporting from shell scripts

---
 test/eventing.bash         |  2 --
 test/lib.bash              | 62 ++++++++++++++++++++++++++++----------
 test/serving.bash          |  3 +-
 test/upstream-e2e-tests.sh |  2 +-
 4 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/test/eventing.bash b/test/eventing.bash
index 5227c5c628..ed9dedd4cf 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -4,7 +4,6 @@ readonly EVENTING_READY_FILE="/tmp/eventing-prober-ready"
 readonly EVENTING_PROBER_FILE="/tmp/eventing-prober-signal"
 
 function upstream_knative_eventing_e2e {
-  (
   logger.info 'Running eventing tests'
 
   local failed=0
@@ -22,7 +21,6 @@ function upstream_knative_eventing_e2e {
   print_test_result ${failed}
 
   return $failed
-  )
 }
 
 function actual_eventing_version {
diff --git a/test/lib.bash b/test/lib.bash
index 5230406899..6307c89190 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -119,7 +119,7 @@ function downstream_eventing_e2e_tests {
 
 function run_rolling_upgrade_tests {
   logger.info "Running rolling upgrade tests"
-  (
+
   local latest_cluster_version latest_serving_version latest_eventing_version \
     rootdir scope serving_in_scope eventing_in_scope serving_prober_pid \
     eventing_prober_pid prev_serving_version prev_eventing_version \
@@ -135,12 +135,21 @@ function run_rolling_upgrade_tests {
   # Save the rootdir before changing dir
   rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")"
 
-  (( serving_in_scope )) && prepare_knative_serving_tests
+  if (( eventing_in_scope )); then
+    prepare_knative_eventing_tests
+  fi
+  if (( serving_in_scope )); then
+    prepare_knative_serving_tests
+  fi
 
   logger.info 'Testing with pre upgrade tests'
 
-  (( serving_in_scope )) && run_serving_preupgrade_test
-  (( eventing_in_scope )) && run_eventing_preupgrade_test
+  if (( serving_in_scope )); then
+    run_serving_preupgrade_test
+  fi
+  if (( eventing_in_scope )); then
+    run_eventing_preupgrade_test
+  fi
 
   logger.info 'Starting prober tests'
 
@@ -153,8 +162,12 @@ function run_rolling_upgrade_tests {
     eventing_prober_pid=$(cat /tmp/prober-pid)
   fi
 
-  (( serving_in_scope )) && wait_for_serving_prober_ready
-  (( eventing_in_scope )) && wait_for_eventing_prober_ready
+  if (( serving_in_scope )); then
+    wait_for_serving_prober_ready
+  fi
+  if (( eventing_in_scope )); then
+    wait_for_eventing_prober_ready
+  fi
 
   if [[ $UPGRADE_SERVERLESS == true ]]; then
     latest_serving_version="${KNATIVE_SERVING_VERSION/v/}"
@@ -165,8 +178,12 @@ function run_rolling_upgrade_tests {
     logger.debug "Eventing version: ${prev_eventing_version} -> ${latest_eventing_version}"
 
     approve_csv "$CURRENT_CSV" "$OLM_UPGRADE_CHANNEL"
-    (( serving_in_scope )) && check_serving_upgraded "${latest_serving_version}"
-    (( eventing_in_scope )) && check_eventing_upgraded "${latest_eventing_version}"
+    if (( serving_in_scope )); then
+      check_serving_upgraded "${latest_serving_version}"
+    fi
+    if (( eventing_in_scope )); then
+      check_eventing_upgraded "${latest_eventing_version}"
+    fi
   fi
 
   # Might not work in OpenShift CI but we want it here so that we can consume
@@ -175,27 +192,40 @@ function run_rolling_upgrade_tests {
     # End the prober test now before we start cluster upgrade, up until now we
     # should have zero failed requests. Cluster upgrade will fail probers as
     # stuff is moved around.
-    (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}"
-    (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}"
+    if (( serving_in_scope )); then
+      end_serving_prober "${serving_prober_pid}"
+    fi
+    if (( eventing_in_scope )); then
+      end_eventing_prober "${eventing_prober_pid}"
+    fi
 
     upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}"
   fi
 
-  (( serving_in_scope )) && wait_for_serving_test_services_settle
+  if (( serving_in_scope )); then
+    wait_for_serving_test_services_settle
+  fi
 
   logger.info "Running postupgrade tests"
 
-  (( serving_in_scope )) && run_serving_postupgrade_test
-  (( eventing_in_scope )) && run_eventing_postupgrade_test
+  if (( serving_in_scope )); then
+    run_serving_postupgrade_test
+  fi
+  if (( eventing_in_scope )); then
+    run_eventing_postupgrade_test
+  fi
 
-  (( serving_in_scope )) && end_serving_prober "${serving_prober_pid}"
-  (( eventing_in_scope )) && end_eventing_prober "${eventing_prober_pid}"
+  if (( serving_in_scope )); then
+    end_serving_prober "${serving_prober_pid}"
+  fi
+  if (( eventing_in_scope )); then
+    end_eventing_prober "${eventing_prober_pid}"
+  fi
 
   cleanup_serving_test_services
 
   cd "$rootdir" || return $?
   return 0
-  )
 }
 
 function end_prober_test {
diff --git a/test/serving.bash b/test/serving.bash
index 4b2debe50c..145698e684 100644
--- a/test/serving.bash
+++ b/test/serving.bash
@@ -33,7 +33,7 @@ function prepare_knative_serving_tests {
 
 function upstream_knative_serving_e2e_and_conformance_tests {
   logger.info "Running Serving E2E and conformance tests"
-  (
+
   prepare_knative_serving_tests || return $?
 
   local failed=0
@@ -76,7 +76,6 @@ function upstream_knative_serving_e2e_and_conformance_tests {
   print_test_result ${failed}
 
   return $failed
-  )
 }
 
 function actual_serving_version {
diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh
index ebee878aad..4da27daf3a 100755
--- a/test/upstream-e2e-tests.sh
+++ b/test/upstream-e2e-tests.sh
@@ -13,7 +13,7 @@ debugging.setup
 
 failed=0
 
-teardown || failed=1
+teardown_serverless || failed=1
 (( !failed )) && create_namespaces || failed=2
 (( !failed )) && install_catalogsource || failed=3
 (( !failed )) && logger.success '🚀 Cluster prepared for testing.'

From 50a8e74389118ad1d03f742dd489f37776fd1cba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Wed, 21 Oct 2020 15:03:44 +0200
Subject: [PATCH 05/13] Remove serving & eventing ns at teardown

---
 hack/lib/serverless.bash   | 6 ++++++
 test/upstream-e2e-tests.sh | 1 +
 2 files changed, 7 insertions(+)

diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash
index 72bc82f566..14a350119d 100644
--- a/hack/lib/serverless.bash
+++ b/hack/lib/serverless.bash
@@ -173,6 +173,9 @@ function teardown_serverless {
   fi
   logger.info 'Ensure no knative serving pods running'
   timeout 600 "[[ \$(oc get pods -n ${SERVING_NAMESPACE} --field-selector=status.phase!=Succeeded -o jsonpath='{.items}') != '[]' ]]" || return 9
+  if oc get namespace "${SERVING_NAMESPACE}" >/dev/null 2>&1; then
+    oc delete namespace "${SERVING_NAMESPACE}"
+  fi
 
   if oc get knativeeventing.operator.knative.dev knative-eventing -n "${EVENTING_NAMESPACE}" >/dev/null 2>&1; then
     logger.info 'Removing KnativeEventing CR'
@@ -180,6 +183,9 @@ function teardown_serverless {
   fi
   logger.info 'Ensure no knative eventing pods running'
   timeout 600 "[[ \$(oc get pods -n ${EVENTING_NAMESPACE} --field-selector=status.phase!=Succeeded -o jsonpath='{.items}') != '[]' ]]" || return 9
+  if oc get namespace "${EVENTING_NAMESPACE}" >/dev/null 2>&1; then
+    oc delete namespace "${EVENTING_NAMESPACE}"
+  fi
 
   oc delete subscriptions.operators.coreos.com -n "${OPERATORS_NAMESPACE}" "${OPERATOR}" 2>/dev/null
   for ip in $(oc get installplan -n "${OPERATORS_NAMESPACE}" | grep serverless-operator | cut -f1 -d' '); do
diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh
index ec38cc265c..83b587359a 100755
--- a/test/upstream-e2e-tests.sh
+++ b/test/upstream-e2e-tests.sh
@@ -22,6 +22,7 @@ teardown_serverless || failed=1
 if [[ $TEST_KNATIVE_UPGRADE == true ]]; then
   # TODO(markusthoemmes): Remove after 1.11 is cut.
   (( !failed )) && oc create namespace "${SERVING_NAMESPACE}"
+  (( !failed )) && oc create namespace "${EVENTING_NAMESPACE}"
 
   (( !failed )) && install_serverless_previous || failed=4
   (( !failed )) && run_rolling_upgrade_tests "${UPGRADE_TEST_SCOPE:-serving,eventing}" || failed=5

From 34cb78cbb98eeaadbb56a5ed40b7cbecc81112ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Mon, 26 Oct 2020 11:17:05 +0100
Subject: [PATCH 06/13] Adding missing return statements + prober end
 idempotency

---
 test/eventing.bash |  30 +++++++++++---
 test/lib.bash      | 100 ++++++++++++++++++++++++++-------------------
 test/serving.bash  |  17 ++++----
 3 files changed, 92 insertions(+), 55 deletions(-)

diff --git a/test/eventing.bash b/test/eventing.bash
index ed9dedd4cf..009654ad93 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -25,7 +25,12 @@ function upstream_knative_eventing_e2e {
 
 function actual_eventing_version {
   oc get knativeeventing.operator.knative.dev \
-    knative-eventing -n "${EVENTING_NAMESPACE}" -o=jsonpath="{.status.version}"
+    knative-eventing -n "${EVENTING_NAMESPACE}" -o=jsonpath="{.status.version}" \
+    || return $?
+}
+
+function prepare_knative_eventing_tests {
+  logger.info 'Nothing to prepare for Eventing upgrade tests'
 }
 
 function run_eventing_preupgrade_test {
@@ -38,7 +43,8 @@ function run_eventing_preupgrade_test {
 
   go_test_e2e -tags=preupgrade \
     -timeout=10m ./test/upgrade \
-    --imagetemplate="${image_template}"
+    --imagetemplate="${image_template}" \
+    || return $?
 
   logger.success 'Eventing pre upgrade tests passed'
 }
@@ -68,7 +74,7 @@ function start_eventing_prober {
 }
 
 function wait_for_eventing_prober_ready {
-  wait_for_file "${EVENTING_READY_FILE}"
+  wait_for_file "${EVENTING_READY_FILE}" || return $?
 
   logger.success 'Eventing prober is ready'
 }
@@ -77,7 +83,20 @@ function end_eventing_prober {
   local prober_pid
   prober_pid="${1:?Pass a prober pid as arg[1]}"
 
-  end_prober_test 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}"
+  end_prober 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}" || return $?
+}
+
+function check_eventing_upgraded {
+  local latest_version
+  latest_version="${1:?Pass a target eventing version as arg[1]}"
+
+  logger.debug 'Check KnativeEventing has the latest version with Ready status'
+  timeout 300 "[[ ! ( \$(oc get knativeeventing.operator.knative.dev \
+    knative-eventing -n ${EVENTING_NAMESPACE} -o=jsonpath='{.status.version}') \
+    == ${latest_version} && \$(oc get knativeeventing.operator.knative.dev \
+    knative-eventing -n ${EVENTING_NAMESPACE} \
+    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" \
+    || return $?
 }
 
 function run_eventing_postupgrade_test {
@@ -90,7 +109,8 @@ function run_eventing_postupgrade_test {
 
   go_test_e2e -tags=postupgrade \
     -timeout=10m ./test/upgrade \
-    --imagetemplate="${image_template}"
+    --imagetemplate="${image_template}" \
+    || return $?
 
   logger.success 'Eventing post upgrade tests passed'
 }
diff --git a/test/lib.bash b/test/lib.bash
index 9b53abc855..bb1e7c327a 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -11,6 +11,8 @@ NAMESPACES+=("${TEST_NAMESPACE}")
 NAMESPACES+=("serverless-tests2")
 NAMESPACES+=("serverless-tests3")
 
+declare -a waited_pids
+
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/serving.bash"
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/eventing.bash"
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/eventing-contrib.bash"
@@ -58,11 +60,11 @@ function serverless_operator_e2e_tests {
   go_test_e2e -failfast -tags=e2e -timeout=30m -parallel=1 ./test/e2e \
     --channel "$OLM_CHANNEL" \
     --kubeconfigs "${kubeconfigs_str}" \
-    "$@" || failed=1
+    "$@" || failed=$?
 
   print_test_result ${failed}
 
-  wait_for_knative_serving_ingress_ns_deleted || return 1
+  wait_for_knative_serving_ingress_ns_deleted || return $?
 
   return $failed
 }
@@ -83,7 +85,7 @@ function serverless_operator_kafka_e2e_tests {
   go_test_e2e -failfast -tags=e2e -timeout=30m -parallel=1 ./test/e2ekafka \
     --channel "$OLM_CHANNEL" \
     --kubeconfigs "${kubeconfigs_str}" \
-    "$@" || failed=1
+    "$@" || failed=$?
 
   print_test_result ${failed}
 
@@ -109,7 +111,7 @@ function downstream_serving_e2e_tests {
   go_test_e2e -failfast -timeout=30m -parallel=1 ./test/servinge2e \
     --kubeconfig "${kubeconfigs[0]}" \
     --kubeconfigs "${kubeconfigs_str}" \
-    "$@" || failed=1
+    "$@" || failed=$?
 
   print_test_result ${failed}
 
@@ -132,7 +134,7 @@ function downstream_knative_kafka_e2e_tests {
   go_test_e2e -failfast -timeout=30m -parallel=1 ./test/extensione2e/kafka \
     --kubeconfig "${kubeconfigs[0]}" \
     --kubeconfigs "${kubeconfigs_str}" \
-    "$@" || failed=1
+    "$@" || failed=$?
 
   print_test_result ${failed}
 
@@ -156,7 +158,7 @@ function downstream_eventing_e2e_tests {
   go_test_e2e -failfast -timeout=30m -parallel=1 ./test/eventinge2e \
     --kubeconfig "${kubeconfigs[0]}" \
     --kubeconfigs "${kubeconfigs_str}" \
-    "$@" || failed=1
+    "$@" || failed=$?
 
   print_test_result ${failed}
 
@@ -171,7 +173,7 @@ function run_rolling_upgrade_tests {
   local latest_cluster_version latest_serving_version latest_eventing_version \
     rootdir scope serving_in_scope eventing_in_scope serving_prober_pid \
     eventing_prober_pid prev_serving_version prev_eventing_version \
-    ocp_target_version
+    ocp_target_version retcode
 
   scope="${1:?Provide an upgrade scope as arg[1]}"
   serving_in_scope="$(echo "${scope}" | grep -vq serving ; echo "$?")"
@@ -184,37 +186,45 @@ function run_rolling_upgrade_tests {
   rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")"
 
   if (( eventing_in_scope )); then
-    prepare_knative_eventing_tests
+    prepare_knative_eventing_tests || return $?
   fi
   if (( serving_in_scope )); then
-    prepare_knative_serving_tests
+    prepare_knative_serving_tests || return $?
   fi
 
   logger.info 'Testing with pre upgrade tests'
 
   if (( serving_in_scope )); then
-    run_serving_preupgrade_test
+    run_serving_preupgrade_test || return $?
   fi
   if (( eventing_in_scope )); then
-    run_eventing_preupgrade_test
+    run_eventing_preupgrade_test || return $?
   fi
 
   logger.info 'Starting prober tests'
 
   if (( serving_in_scope )); then
     start_serving_prober "${prev_serving_version}" /tmp/prober-pid
+    retcode=$?
     serving_prober_pid=$(cat /tmp/prober-pid)
+    if (( retcode )); then
+      return $retcode
+    fi
   fi
   if (( eventing_in_scope )); then
     start_eventing_prober /tmp/prober-pid
+    retcode=$?
     eventing_prober_pid=$(cat /tmp/prober-pid)
+    if (( retcode )); then
+      return $retcode
+    fi
   fi
 
   if (( serving_in_scope )); then
-    wait_for_serving_prober_ready
+    wait_for_serving_prober_ready || return $?
   fi
   if (( eventing_in_scope )); then
-    wait_for_eventing_prober_ready
+    wait_for_eventing_prober_ready || return $?
   fi
 
   if [[ $UPGRADE_SERVERLESS == true ]]; then
@@ -227,10 +237,10 @@ function run_rolling_upgrade_tests {
 
     approve_csv "$CURRENT_CSV" "$OLM_UPGRADE_CHANNEL"
     if (( serving_in_scope )); then
-      check_serving_upgraded "${latest_serving_version}"
+      check_serving_upgraded "${latest_serving_version}" || return $?
     fi
     if (( eventing_in_scope )); then
-      check_eventing_upgraded "${latest_eventing_version}"
+      check_eventing_upgraded "${latest_eventing_version}" || return $?
     fi
   fi
 
@@ -241,47 +251,51 @@ function run_rolling_upgrade_tests {
     # should have zero failed requests. Cluster upgrade will fail probers as
     # stuff is moved around.
     if (( serving_in_scope )); then
-      end_serving_prober "${serving_prober_pid}"
+      end_serving_prober "${serving_prober_pid}" || return $?
     fi
     if (( eventing_in_scope )); then
-      end_eventing_prober "${eventing_prober_pid}"
+      end_eventing_prober "${eventing_prober_pid}" || return $?
     fi
 
-    upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}"
+    upgrade_ocp_cluster "${UPGRADE_OCP_IMAGE:-}" || return $?
   fi
 
   if (( serving_in_scope )); then
-    wait_for_serving_test_services_settle
+    wait_for_serving_test_services_settle || return $?
   fi
 
   logger.info "Running postupgrade tests"
 
   if (( serving_in_scope )); then
-    run_serving_postupgrade_test
+    run_serving_postupgrade_test || return $?
   fi
   if (( eventing_in_scope )); then
-    run_eventing_postupgrade_test
+    run_eventing_postupgrade_test || return $?
   fi
 
   if (( serving_in_scope )); then
-    end_serving_prober "${serving_prober_pid}"
+    end_serving_prober "${serving_prober_pid}" || return $?
   fi
   if (( eventing_in_scope )); then
-    end_eventing_prober "${eventing_prober_pid}"
+    end_eventing_prober "${eventing_prober_pid}" || return $?
   fi
 
-  cleanup_serving_test_services
+  cleanup_serving_test_services || return $?
 
   cd "$rootdir" || return $?
   return 0
 }
 
-function end_prober_test {
+function end_prober {
   local prober_pid prober_signal retcode title
   title=${1:?Pass a title as arg[1]}
   prober_pid=${2:?Pass a pid as a arg[2]}
   prober_signal=${3:-/tmp/prober-signal}
 
+  if array.contains "${prober_pid}" "${waited_pids[@]}"; then
+    logger.info "Prober of PID ${prober_pid} is closed already."
+    return 0
+  fi
   if kill -0 "${prober_pid}" 2>/dev/null; then
     logger.debug "${title} prober of PID ${prober_pid} isn't running..."
   else
@@ -291,6 +305,7 @@ function end_prober_test {
 
   wait "${prober_pid}"
   retcode=$?
+  waited_pids+=("${prober_pid}")
   if ! (( retcode )); then
     logger.success "${title} prober passed"
   else
@@ -306,15 +321,15 @@ function upgrade_ocp_cluster {
   if [[ -n "$upgrade_ocp_image" ]]; then
     ocp_target_version="$upgrade_ocp_image"
     oc adm upgrade --to-image="${UPGRADE_OCP_IMAGE}" \
-      --force=true --allow-explicit-upgrade
+      --force=true --allow-explicit-upgrade || return $?
   else
     latest_cluster_version=$(oc adm upgrade | sed -ne '/VERSION/,$ p' \
       | grep -v VERSION | awk '{print $1}' | sort -r | head -n 1)
     [[ $latest_cluster_version != "" ]] || return 1
     ocp_target_version="$latest_cluster_version"
-    oc adm upgrade --to-latest=true --force=true
+    oc adm upgrade --to-latest=true --force=true || return $?
   fi
-  timeout 7200 "[[ \$(oc get clusterversion version -o jsonpath='{.status.history[?(@.image==\"${ocp_target_version}\")].state}') != Completed ]]" || return 1
+  timeout 7200 "[[ \$(oc get clusterversion version -o jsonpath='{.status.history[?(@.image==\"${ocp_target_version}\")].state}') != Completed ]]" || return $?
 
   logger.success "New cluster version: $(oc get clusterversion \
     version -o jsonpath='{.status.desired.version}')"
@@ -380,28 +395,28 @@ function create_htpasswd_users {
 
   logger.info 'Add users to htpasswd'
   for i in $(seq 1 $num_users); do
-    htpasswd -b users.htpasswd "user${i}" "password${i}"
+    htpasswd -b users.htpasswd "user${i}" "password${i}" || return $?
   done
 
   kubectl create secret generic htpass-secret \
     --from-file=htpasswd="$(pwd)/users.htpasswd" \
     -n openshift-config \
-    --dry-run -o yaml | kubectl apply -f -
-  oc apply -f openshift/identity/htpasswd.yaml
+    --dry-run -o yaml | kubectl apply -f -  || return $?
+  oc apply -f openshift/identity/htpasswd.yaml || return $?
 
   logger.info 'Generate kubeconfig for each user'
   for i in $(seq 1 $num_users); do
     cp "${KUBECONFIG}" "user${i}.kubeconfig"
     occmd="bash -c '! oc login --kubeconfig=user${i}.kubeconfig --username=user${i} --password=password${i} > /dev/null'"
-    timeout 180 "${occmd}" || return 1
+    timeout 180 "${occmd}" || return $?
   done
 }
 
 function add_roles {
   logger.info "Adding roles to users"
-  oc adm policy add-role-to-user admin user1 -n "$TEST_NAMESPACE"
-  oc adm policy add-role-to-user edit user2 -n "$TEST_NAMESPACE"
-  oc adm policy add-role-to-user view user3 -n "$TEST_NAMESPACE"
+  oc adm policy add-role-to-user admin user1 -n "$TEST_NAMESPACE" || return $?
+  oc adm policy add-role-to-user edit user2 -n "$TEST_NAMESPACE" || return $?
+  oc adm policy add-role-to-user view user3 -n "$TEST_NAMESPACE" || return $?
 }
 
 function delete_users {
@@ -411,10 +426,10 @@ function delete_users {
     logger.debug "htpasswd user line: ${line}"
     user=$(echo "${line}" | cut -d: -f1)
     if [ -f "${user}.kubeconfig" ]; then
-      rm -v "${user}.kubeconfig"
+      rm -fv "${user}.kubeconfig" || return $?
     fi
   done < "users.htpasswd"
-  rm -v users.htpasswd
+  rm -fv users.htpasswd  || return $?
 }
 
 function add_systemnamespace_label {
@@ -423,8 +438,8 @@ function add_systemnamespace_label {
 }
 
 function add_networkpolicy {
-  local NAMESPACE=$1
-  cat <<EOF | oc apply -f -
+  cat <<EOF | oc apply -f - || return $?
+---
 kind: NetworkPolicy
 apiVersion: networking.k8s.io/v1
 metadata:
@@ -432,9 +447,7 @@ metadata:
   namespace: "$1"
 spec:
   podSelector:
-EOF
-
-  cat <<EOF | oc apply -f -
+---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
@@ -479,7 +492,8 @@ function trigger_gc_and_print_knative {
 function wait_for_leader_controller() {
   echo -n "Waiting for a leader Controller"
   for i in {1..150}; do  # timeout after 5 minutes
-    local leader=$(oc get lease -n "${SERVING_NAMESPACE}" -ojsonpath='{range .items[*].spec}{"\n"}{.holderIdentity}' | cut -d"_" -f1 | grep "^controller-" | head -1)
+    local leader
+    leader=$(oc get lease -n "${SERVING_NAMESPACE}" -ojsonpath='{range .items[*].spec}{"\n"}{.holderIdentity}' | cut -d"_" -f1 | grep "^controller-" | head -1)
     # Make sure the leader pod exists.
     if [ -n "${leader}" ] && oc get pod "${leader}" -n "${SERVING_NAMESPACE}"  >/dev/null 2>&1; then
       echo -e "\nNew leader Controller has been elected"
diff --git a/test/serving.bash b/test/serving.bash
index 068aea1b66..4dd427bfc4 100644
--- a/test/serving.bash
+++ b/test/serving.bash
@@ -106,7 +106,8 @@ function upstream_knative_serving_e2e_and_conformance_tests {
 
 function actual_serving_version {
   oc get knativeserving.operator.knative.dev \
-    knative-serving -n "${SERVING_NAMESPACE}" -o=jsonpath="{.status.version}"
+    knative-serving -n "${SERVING_NAMESPACE}" -o=jsonpath="{.status.version}" \
+    || return $?
 }
 
 function run_serving_preupgrade_test {
@@ -121,7 +122,7 @@ function run_serving_preupgrade_test {
   go_test_e2e -tags=preupgrade -timeout=20m ./test/upgrade \
     --imagetemplate "$image_template" \
     --kubeconfig "$KUBECONFIG" \
-    --resolvabledomain
+    --resolvabledomain || return $?
 
   logger.success 'Serving pre upgrade tests passed'
 }
@@ -163,7 +164,7 @@ function wait_for_serving_prober_ready {
   # Wait for the upgrade-probe kservice to be ready before proceeding
   timeout 900 "[[ \$(oc get services.serving.knative.dev upgrade-probe \
     -n serving-tests -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') \
-    != True ]]"
+    != True ]]" || return $?
 
   logger.success 'Serving prober is ready'
 }
@@ -177,21 +178,22 @@ function check_serving_upgraded {
     knative-serving -n ${SERVING_NAMESPACE} -o=jsonpath='{.status.version}') \
     == ${latest_serving_version} && \$(oc get knativeserving.operator.knative.dev \
     knative-serving -n ${SERVING_NAMESPACE} \
-    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]"
+    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" \
+    || return $?
 }
 
 function end_serving_prober {
   local prober_pid
   prober_pid="${1:?Pass a prober pid as arg[1]}"
 
-  end_prober_test 'Serving' "${prober_pid}"
+  end_prober 'Serving' "${prober_pid}" || return $?
 }
 
 function wait_for_serving_test_services_settle {
   # Wait for all services to become ready again. Exclude the upgrade-probe as
   # that'll be removed by the prober test above.
   for kservice in $(oc get ksvc -n serving-tests --no-headers -o name | grep -v 'upgrade-probe'); do
-    timeout 900 "[[ \$(oc get ${kservice} -n serving-tests -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]"
+    timeout 900 "[[ \$(oc get ${kservice} -n serving-tests -o jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]" || return $?
   done
 
   # Give time to settle things down
@@ -220,5 +222,6 @@ function cleanup_serving_test_services {
   oc delete --ignore-not-found=true ksvc \
     pizzaplanet-upgrade-service \
     scale-to-zero-upgrade-service \
-    upgrade-probe -n serving-tests
+    upgrade-probe -n serving-tests \
+    || return $?
 }

From af0f0105356406430d5f3a5cc992c42be7178167 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Mon, 26 Oct 2020 13:16:54 +0100
Subject: [PATCH 07/13] Declare a waited PIDs array

---
 test/lib.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/lib.bash b/test/lib.bash
index bb1e7c327a..8ff1c45c00 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -11,7 +11,7 @@ NAMESPACES+=("${TEST_NAMESPACE}")
 NAMESPACES+=("serverless-tests2")
 NAMESPACES+=("serverless-tests3")
 
-declare -a waited_pids
+declare -a waited_pids=()
 
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/serving.bash"
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/eventing.bash"

From bc359474125204810227061f5cc27edd584df804 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Mon, 26 Oct 2020 17:18:49 +0100
Subject: [PATCH 08/13] Rely on PID directory instead of Bash arrays

---
 hack/lib/common.bash |  7 -------
 test/lib.bash        | 11 ++++++-----
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/hack/lib/common.bash b/hack/lib/common.bash
index bf0efab00c..0b96cfcbf7 100644
--- a/hack/lib/common.bash
+++ b/hack/lib/common.bash
@@ -1,12 +1,5 @@
 #!/usr/bin/env bash
 
-function array.contains {
-  local e match="$1"
-  shift
-  for e; do [[ "$e" == "$match" ]] && return 0; done
-  return 1
-}
-
 function array.join {
   local IFS="$1"
   shift
diff --git a/test/lib.bash b/test/lib.bash
index 8ff1c45c00..dd3f176177 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -11,8 +11,6 @@ NAMESPACES+=("${TEST_NAMESPACE}")
 NAMESPACES+=("serverless-tests2")
 NAMESPACES+=("serverless-tests3")
 
-declare -a waited_pids=()
-
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/serving.bash"
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/eventing.bash"
 source "$(dirname "$(realpath "${BASH_SOURCE[0]}")")/eventing-contrib.bash"
@@ -287,12 +285,15 @@ function run_rolling_upgrade_tests {
 }
 
 function end_prober {
-  local prober_pid prober_signal retcode title
+  local prober_pid prober_signal retcode title piddir
   title=${1:?Pass a title as arg[1]}
   prober_pid=${2:?Pass a pid as a arg[2]}
   prober_signal=${3:-/tmp/prober-signal}
+  piddir="${piddir:-/tmp/svls-probes/$$}"
+
+  mkdir -p "${piddir}" || return $?
 
-  if array.contains "${prober_pid}" "${waited_pids[@]}"; then
+  if [ -f "${piddir}/${prober_pid}" ]; then
     logger.info "Prober of PID ${prober_pid} is closed already."
     return 0
   fi
@@ -305,7 +306,7 @@ function end_prober {
 
   wait "${prober_pid}"
   retcode=$?
-  waited_pids+=("${prober_pid}")
+  echo 'done' > "${piddir}/${prober_pid}"
   if ! (( retcode )); then
     logger.success "${title} prober passed"
   else

From d402d57389271d8a2fd3a7efb895e7fb5e5c6580 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Tue, 27 Oct 2020 17:01:06 +0100
Subject: [PATCH 09/13] Eventing test bits deploy properly

---
 test/eventing.bash | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/eventing.bash b/test/eventing.bash
index aefa5a8f72..3e5f67273f 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -37,7 +37,8 @@ function run_eventing_preupgrade_test {
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
   local image_template
-  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+  # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
+  image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
 
   go_test_e2e -tags=preupgrade \
     -timeout=10m ./test/upgrade \
@@ -56,7 +57,8 @@ function start_eventing_prober {
   rm -fv "${EVENTING_PROBER_FILE}" "${EVENTING_READY_FILE}"
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
-  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+  # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
+  image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
 
   go_test_e2e -tags=probe \
     -timeout=30m \
@@ -103,7 +105,8 @@ function run_eventing_postupgrade_test {
 
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
-  image_template="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+  # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
+  image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
 
   go_test_e2e -tags=postupgrade \
     -timeout=10m ./test/upgrade \
@@ -112,5 +115,3 @@ function run_eventing_postupgrade_test {
 
   logger.success 'Eventing post upgrade tests passed'
 }
-
-

From 14759d15f71937b73228b18d36c0c95e858a0cbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Tue, 27 Oct 2020 20:10:53 +0100
Subject: [PATCH 10/13] Proper waiting on prober's end

---
 test/lib.bash | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/test/lib.bash b/test/lib.bash
index 02304fae1c..49e90d7646 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -310,12 +310,8 @@ function end_prober {
     logger.info "Prober of PID ${prober_pid} is closed already."
     return 0
   fi
-  if kill -0 "${prober_pid}" 2>/dev/null; then
-    logger.debug "${title} prober of PID ${prober_pid} isn't running..."
-  else
-    echo 'done' > "${prober_signal}"
-    logger.info "Waiting for ${title} prober test to finish"
-  fi
+  logger.info "Waiting for ${title} prober test to finish"
+  echo 'done' > "${prober_signal}"
 
   wait "${prober_pid}"
   retcode=$?

From 30b91b9da466f1df7b11c6324866c8c610d526ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Tue, 27 Oct 2020 20:17:34 +0100
Subject: [PATCH 11/13] Creating a namespaces for previous release

---
 hack/lib/common.bash       | 21 +++++++++++++++++++++
 hack/lib/namespaces.bash   | 12 +++++++++---
 hack/lib/serverless.bash   | 10 ++++++++++
 test/upstream-e2e-tests.sh |  4 ----
 4 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/hack/lib/common.bash b/hack/lib/common.bash
index 866fa39c40..8f81129fd9 100644
--- a/hack/lib/common.bash
+++ b/hack/lib/common.bash
@@ -43,3 +43,24 @@ function wait_for_file {
 
   timeout "${waits}" "[[ ! -f '${file}' ]]" || return $?
 }
+
+function versions.le {
+  local v1 v2 cmp
+  v1="${1:?Pass a version to check as arg[1]}"
+  v2="${2:?Pass a version to check against as arg[2]}"
+  cmp="$(echo -e "${v1}\n${v2}" | sort -V | head -n 1)"
+
+  [ "${v1}" = "${cmp}" ]
+}
+
+function versions.lt {
+  local v1 v2
+  v1="${1:?Pass a version to check as arg[1]}"
+  v2="${2:?Pass a version to check against as arg[2]}"
+
+  if ! [ "${v1}" = "${v2}" ]; then
+    return 1
+  fi
+
+  versions.le "${v1}" "${v2}"
+}
diff --git a/hack/lib/namespaces.bash b/hack/lib/namespaces.bash
index 6b4b219156..e078fcbf89 100644
--- a/hack/lib/namespaces.bash
+++ b/hack/lib/namespaces.bash
@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 
+function ensure_namespace {
+  local ns
+  ns="${1:?Pass namespace name as arg[1]}"
+  if ! oc get namespace "${ns}" >/dev/null 2>&1; then
+    oc create namespace "${ns}"
+  fi
+}
+
 function create_namespaces {
   logger.info 'Create namespaces'
   for ns in "${NAMESPACES[@]}"; do
-    if ! oc get ns "${ns}" >/dev/null 2>&1; then
-      oc create ns "${ns}"
-    fi
+    ensure_namespace "${ns}"
   done
   # Create an OperatorGroup if there are no other ones in the namespace.
   if [[ $(oc get operatorgroups -oname -n "${OPERATORS_NAMESPACE}" | wc -l) -eq 0 ]]; then
diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash
index 8e7d59bbfe..14d5af18fa 100644
--- a/hack/lib/serverless.bash
+++ b/hack/lib/serverless.bash
@@ -30,6 +30,16 @@ function install_serverless_previous {
   remove_installplan "$CURRENT_CSV"
 
   deploy_serverless_operator "$PREVIOUS_CSV"  || return $?
+
+  # TODO(ksuszyns): Remove this if block if no longer required
+  if versions.le "$(metadata.get olm.replaces)" 1.11.0; then
+    logger.info "Ensure ${SERVING_NAMESPACE} and ${EVENTING_NAMESPACE} \
+namespaces exists, as ${PREVIOUS_CSV} didn't created them automatically."
+
+    ensure_namespace "${SERVING_NAMESPACE}"
+    ensure_namespace "${EVENTING_NAMESPACE}"
+  fi
+
   deploy_knativeserving_cr || return $?
   deploy_knativeeventing_cr || return $?
   logger.success "Previous version of Serverless is installed: $PREVIOUS_CSV"
diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh
index be7133f73d..271b5ec1fe 100755
--- a/test/upstream-e2e-tests.sh
+++ b/test/upstream-e2e-tests.sh
@@ -20,10 +20,6 @@ teardown_serverless || failed=1
 
 # Run upgrade tests
 if [[ $TEST_KNATIVE_UPGRADE == true ]]; then
-  # TODO(markusthoemmes): Remove after 1.11 is cut.
-  (( !failed )) && oc create namespace "${SERVING_NAMESPACE}"
-  (( !failed )) && oc create namespace "${EVENTING_NAMESPACE}"
-
   (( !failed )) && install_serverless_previous || failed=4
   (( !failed )) && run_rolling_upgrade_tests "${UPGRADE_TEST_SCOPE:-serving,eventing}" || failed=5
   (( !failed )) && trigger_gc_and_print_knative || failed=6

From 6cfb729868de4969e3cbb9a8ba2fcc1c310748f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Fri, 30 Oct 2020 14:49:49 +0100
Subject: [PATCH 12/13] Add env vars to configure upgrade tests

---
 test/eventing.bash | 16 ++++++++++++----
 test/serving.bash  |  6 +++---
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/test/eventing.bash b/test/eventing.bash
index 3e5f67273f..afafa0a5d2 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -49,17 +49,25 @@ function run_eventing_preupgrade_test {
 }
 
 function start_eventing_prober {
-  local eventing_prober_pid result_file image_template
-  result_file="${1:?Pass a result file as arg[1]}"
-
+  local eventing_prober_pid pid_file image_template eventing_prober_interval
+  pid_file="${1:?Pass a PID file as arg[1]}"
   logger.info 'Starting Eventing prober'
 
+  EVENTING_PROBER_INTERVAL_MSEC="${EVENTING_PROBER_INTERVAL_MSEC:-2}"
+  eventing_prober_interval="${EVENTING_PROBER_INTERVAL_MSEC}ms"
+
+
   rm -fv "${EVENTING_PROBER_FILE}" "${EVENTING_READY_FILE}"
   cd "${KNATIVE_EVENTING_HOME}" || return $?
 
   # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
   image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
 
+  # FIXME: knative/operator#297 Restore scale to zero setting
+  E2E_UPGRADE_TESTS_SERVING_SCALETOZERO=false \
+  E2E_UPGRADE_TESTS_SERVING_USE=true \
+  E2E_UPGRADE_TESTS_CONFIGMOUNTPOINT=/.config/wathola \
+  E2E_UPGRADE_TESTS_INTERVAL="${eventing_prober_interval}" \
   go_test_e2e -tags=probe \
     -timeout=30m \
     ./test/upgrade \
@@ -70,7 +78,7 @@ function start_eventing_prober {
 
   logger.debug "Eventing prober PID is ${eventing_prober_pid}"
 
-  echo ${eventing_prober_pid} > "${result_file}"
+  echo ${eventing_prober_pid} > "${pid_file}"
 }
 
 function wait_for_eventing_prober_ready {
diff --git a/test/serving.bash b/test/serving.bash
index 8443f54dff..5f4a7c516f 100644
--- a/test/serving.bash
+++ b/test/serving.bash
@@ -127,9 +127,9 @@ function run_serving_preupgrade_test {
 
 function start_serving_prober {
   local image_template prev_serving_version probe_fraction serving_prober_pid \
-    result_file
+    pid_file
   prev_serving_version="${1:?Pass a previous Serving version as arg[1]}"
-  result_file="${2:?Pass a result file as arg[2]}"
+  pid_file="${2:?Pass a PID file as arg[2]}"
 
   logger.info 'Starting Serving prober'
 
@@ -155,7 +155,7 @@ function start_serving_prober {
 
   logger.debug "Serving prober PID is ${serving_prober_pid}"
 
-  echo ${serving_prober_pid} > "${result_file}"
+  echo ${serving_prober_pid} > "${pid_file}"
 }
 
 function wait_for_serving_prober_ready {

From 7c2b75edbaeaadeb5cf90c258dc15f66d16b1ce9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Mon, 2 Nov 2020 00:08:21 +0100
Subject: [PATCH 13/13] Lowering interval from 2ms to 50ms

To avoid loosing events and most notably "finished" event, I'm lowering
interval between sent event to 50 ms from previously used 2 ms.
---
 test/eventing.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/eventing.bash b/test/eventing.bash
index afafa0a5d2..2933c1a468 100644
--- a/test/eventing.bash
+++ b/test/eventing.bash
@@ -53,7 +53,7 @@ function start_eventing_prober {
   pid_file="${1:?Pass a PID file as arg[1]}"
   logger.info 'Starting Eventing prober'
 
-  EVENTING_PROBER_INTERVAL_MSEC="${EVENTING_PROBER_INTERVAL_MSEC:-2}"
+  EVENTING_PROBER_INTERVAL_MSEC="${EVENTING_PROBER_INTERVAL_MSEC:-50}"
   eventing_prober_interval="${EVENTING_PROBER_INTERVAL_MSEC}ms"