Eventing upgrade tests

hack/lib/catalogsource.bash

@@ -13,6 +13,7 @@ function install_catalogsource {
   logger.info "Installing CatalogSource"
 
   local rootdir pull_user
+
   rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
 
   # Add a user that is allowed to pull images from the registry.
@@ -44,15 +45,15 @@ function install_catalogsource {
   if ! oc get buildconfigs serverless-bundle -n "$OLM_NAMESPACE" >/dev/null 2>&1; then
     logger.info 'Create a bundle image build'
     oc -n "$OLM_NAMESPACE" new-build --binary \
-      --strategy=docker --name serverless-bundle
+      --strategy=docker --name serverless-bundle || return $?
   else
     logger.info 'Serverless bundle image build is already created'
   fi
   if ! [ -f "${rootdir}/_output/serverless-bundle.sha1sum" ] || \
       ! sha1sum --check --status "${rootdir}/_output/serverless-bundle.sha1sum"; then
     logger.info 'Build the bundle image in the cluster-internal registry.'
     oc -n "$OLM_NAMESPACE" start-build serverless-bundle \
-      --from-dir "${rootdir}/olm-catalog/serverless-operator" -F
+      --from-dir "${rootdir}/olm-catalog/serverless-operator" -F  || return $?
     mkdir -p "${rootdir}/_output"
     find "${rootdir}/olm-catalog/serverless-operator" -type f -exec sha1sum {} + \
       > "${rootdir}/_output/serverless-bundle.sha1sum"

hack/lib/common.bash

@@ -1,12 +1,5 @@
 #!/usr/bin/env bash
 
-function array.contains {
-  local e match="$1"
-  shift
-  for e; do [[ "$e" == "$match" ]] && return 0; done
-  return 1
-}
-
 function array.join {
   local IFS="$1"
   shift
@@ -42,3 +35,32 @@ function timeout {
   done
   return 0
 }
+
+function wait_for_file {
+  local file timeout waits
+  file="${1:?Pass a filepath as arg[1]}"
+  waits="${2:-300}"
+
+  timeout "${waits}" "[[ ! -f '${file}' ]]" || return $?
+}
+
+function versions.le {
+  local v1 v2 cmp
+  v1="${1:?Pass a version to check as arg[1]}"
+  v2="${2:?Pass a version to check against as arg[2]}"
+  cmp="$(echo -e "${v1}\n${v2}" | sort -V | head -n 1)"
+
+  [ "${v1}" = "${cmp}" ]
+}
+
+function versions.lt {
+  local v1 v2
+  v1="${1:?Pass a version to check as arg[1]}"
+  v2="${2:?Pass a version to check against as arg[2]}"
+
+  if ! [ "${v1}" = "${v2}" ]; then
+    return 1
+  fi
+
+  versions.le "${v1}" "${v2}"
+}

hack/lib/namespaces.bash

@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 
+function ensure_namespace {
+  local ns
+  ns="${1:?Pass namespace name as arg[1]}"
+  if ! oc get namespace "${ns}" >/dev/null 2>&1; then
+    oc create namespace "${ns}"
+  fi
+}
+
 function create_namespaces {
   logger.info 'Create namespaces'
   for ns in "${NAMESPACES[@]}"; do
-    if ! oc get ns "${ns}" >/dev/null 2>&1; then
-      oc create ns "${ns}"
-    fi
+    ensure_namespace "${ns}"
   done
   # Create an OperatorGroup if there are no other ones in the namespace.
   if [[ $(oc get operatorgroups -oname -n "${OPERATORS_NAMESPACE}" | wc -l) -eq 0 ]]; then

hack/lib/serverless.bash

@@ -23,26 +23,30 @@ function ensure_serverless_installed {
 }
 
 function install_serverless_previous {
-  local rootdir
-  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
+  logger.info "Installing previous version of Serverless..."
 
   # Remove installplan from previous installations, leaving this would make the operator
   # upgrade to the latest version immediately
   remove_installplan "$CURRENT_CSV"
 
   deploy_serverless_operator "$PREVIOUS_CSV"  || return $?
-  deploy_knativeserving_cr || return $?
-}
 
-function remove_installplan {
-  local install_plan
-  install_plan=$(find_install_plan $1)
-  if [[ -n $install_plan ]]; then
-    oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}"
+  # TODO(ksuszyns): Remove this if block if no longer required
+  if versions.le "$(metadata.get olm.replaces)" 1.11.0; then
+    logger.info "Ensure ${SERVING_NAMESPACE} and ${EVENTING_NAMESPACE} \
+namespaces exists, as ${PREVIOUS_CSV} didn't created them automatically."
+
+    ensure_namespace "${SERVING_NAMESPACE}"
+    ensure_namespace "${EVENTING_NAMESPACE}"
   fi
+
+  deploy_knativeserving_cr || return $?
+  deploy_knativeeventing_cr || return $?
+  logger.success "Previous version of Serverless is installed: $PREVIOUS_CSV"
 }
 
 function install_serverless_latest {
+  logger.info "Installing latest version of Serverless..."
   deploy_serverless_operator_latest || return $?
 
   if [[ $INSTALL_SERVING == "true" ]]; then
@@ -54,20 +58,32 @@ function install_serverless_latest {
   if [[ $INSTALL_KAFKA == "true" ]]; then
     deploy_knativekafka_cr || return $?
   fi
+
+  logger.success "Latest version of Serverless is installed: $CURRENT_CSV"
+}
+
+function remove_installplan {
+  local install_plan csv
+  csv="${1:?Pass a CSV as arg[1]}"
+  logger.info "Removing installplan for $csv"
+  install_plan=$(find_install_plan "$csv")
+  if [[ -n $install_plan ]]; then
+    oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}"
+  else
+    logger.debug "No install plan for $csv"
+  fi
 }
 
 function deploy_serverless_operator_latest {
-  local rootdir
-  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
   deploy_serverless_operator "$CURRENT_CSV"
 }
 
 function deploy_serverless_operator {
-  local csv
-  csv="$1"
-  logger.info "Install the Serverless Operator ${csv}"
-
-  cat <<EOF | oc apply -f - || return $?
+  local csv tmpfile
+  csv="${1:?Pass as CSV as arg[1]}"
+  logger.info "Install the Serverless Operator: ${csv}"
+  tmpfile=$(mktemp /tmp/subscription.XXXXXX.yaml)
+  cat > "$tmpfile" <<EOF
 apiVersion: operators.coreos.com/v1alpha1
 kind: Subscription
 metadata:
@@ -81,23 +97,25 @@ spec:
   installPlanApproval: Manual
   startingCSV: "${csv}"
 EOF
+  [ -n "$OPENSHIFT_CI" ] && cat "$tmpfile"
+  oc apply -f "$tmpfile" || return $?
 
   # Approve the initial installplan automatically
   approve_csv "$csv" "$OLM_CHANNEL" || return 5
 }
 
 function approve_csv {
   local csv_version install_plan channel
-  csv_version=$1
-  channel=$2
+  csv_version=${1:?Pass a CSV as arg[1]}
+  channel=${2:?Pass channel as arg[2]}
 
-  # Ensure channel and source is set properly
+  logger.info 'Ensure channel and source is set properly'
   oc patch subscriptions.operators.coreos.com "$OPERATOR" -n "${OPERATORS_NAMESPACE}" \
     --type 'merge' \
     --patch '{"spec": {"channel": "'"${channel}"'", "source": "'"${OLM_SOURCE}"'"}}' \
     || return $?
 
-  # Wait for the installplan to be available
+  logger.info 'Wait for the installplan to be available'
   timeout 900 "[[ -z \$(find_install_plan $csv_version) ]]" || return 1
 
   install_plan=$(find_install_plan $csv_version)
@@ -195,6 +213,9 @@ function teardown_serverless {
   fi
   logger.info 'Ensure no knative serving pods running'
   timeout 600 "[[ \$(oc get pods -n ${SERVING_NAMESPACE} --field-selector=status.phase!=Succeeded -o jsonpath='{.items}') != '[]' ]]" || return 9
+  if oc get namespace "${SERVING_NAMESPACE}" >/dev/null 2>&1; then
+    oc delete namespace "${SERVING_NAMESPACE}"
+  fi
 
   if oc get knativeeventing.operator.knative.dev knative-eventing -n "${EVENTING_NAMESPACE}" >/dev/null 2>&1; then
     logger.info 'Removing KnativeEventing CR'
@@ -208,6 +229,9 @@ function teardown_serverless {
   fi
   logger.info 'Ensure no knative eventing or knative kafka pods running'
   timeout 600 "[[ \$(oc get pods -n ${EVENTING_NAMESPACE} --field-selector=status.phase!=Succeeded -o jsonpath='{.items}') != '[]' ]]" || return 10
+  if oc get namespace "${EVENTING_NAMESPACE}" >/dev/null 2>&1; then
+    oc delete namespace "${EVENTING_NAMESPACE}"
+  fi
 
   oc delete subscriptions.operators.coreos.com -n "${OPERATORS_NAMESPACE}" "${OPERATOR}" 2>/dev/null
   for ip in $(set +o pipefail && oc get installplan -n "${OPERATORS_NAMESPACE}" | grep serverless-operator | cut -f1 -d' '); do

hack/lib/vars.bash

@@ -53,7 +53,8 @@ export INSTALL_PREVIOUS_VERSION="${INSTALL_PREVIOUS_VERSION:-"false"}"
 
 
 # Using first channel on the list, instead of default one
-export OLM_CHANNEL="${OLM_CHANNEL:-$(metadata.get 'olm.channels.list[*]' | head -n 1)}"
+OLM_CHANNEL="${OLM_CHANNEL:-$(metadata.get 'olm.channels.list[*]' | head -n 1)}"
+export OLM_CHANNEL
 # Change this when upgrades need switching to a different channel
 export OLM_UPGRADE_CHANNEL="${OLM_UPGRADE_CHANNEL:-"$OLM_CHANNEL"}"
 export OLM_SOURCE="${OLM_SOURCE:-"$OPERATOR"}"

test/eventing.bash

@@ -1,20 +1,125 @@
 #!/usr/bin/env bash
 
+readonly EVENTING_READY_FILE="/tmp/eventing-prober-ready"
+readonly EVENTING_PROBER_FILE="/tmp/eventing-prober-signal"
+
 function upstream_knative_eventing_e2e {
-  (
   logger.info 'Running eventing tests'
 
   local failed=0
 
-  TEST_IMAGE_TEMPLATE="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
+  export TEST_IMAGE_TEMPLATE="registry.svc.ci.openshift.org/openshift/knative-${KNATIVE_EVENTING_VERSION}:knative-eventing-test-{{.Name}}"
 
-  cd "$KNATIVE_EVENTING_HOME" || return $?
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
 
+  # shellcheck disable=SC1090
   source "${KNATIVE_EVENTING_HOME}/openshift/e2e-common.sh"
 
   # run_e2e_tests defined in knative-eventing
   run_e2e_tests || failed=$?
 
   return $failed
-  )
+}
+
+function actual_eventing_version {
+  oc get knativeeventing.operator.knative.dev \
+    knative-eventing -n "${EVENTING_NAMESPACE}" -o=jsonpath="{.status.version}" \
+    || return $?
+}
+
+function prepare_knative_eventing_tests {
+  logger.info 'Nothing to prepare for Eventing upgrade tests'
+}
+
+function run_eventing_preupgrade_test {
+  logger.info 'Running Eventing pre upgrade tests'
+
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
+
+  local image_template
+  # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
+  image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
+
+  go_test_e2e -tags=preupgrade \
+    -timeout=10m ./test/upgrade \
+    --imagetemplate="${image_template}" \
+    || return $?
+
+  logger.success 'Eventing pre upgrade tests passed'
+}
+
+function start_eventing_prober {
+  local eventing_prober_pid pid_file image_template eventing_prober_interval
+  pid_file="${1:?Pass a PID file as arg[1]}"
+  logger.info 'Starting Eventing prober'
+
+  EVENTING_PROBER_INTERVAL_MSEC="${EVENTING_PROBER_INTERVAL_MSEC:-50}"
+  eventing_prober_interval="${EVENTING_PROBER_INTERVAL_MSEC}ms"
+
+
+  rm -fv "${EVENTING_PROBER_FILE}" "${EVENTING_READY_FILE}"
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
+
+  # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
+  image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
+
+  # FIXME: knative/operator#297 Restore scale to zero setting
+  E2E_UPGRADE_TESTS_SERVING_SCALETOZERO=false \
+  E2E_UPGRADE_TESTS_SERVING_USE=true \
+  E2E_UPGRADE_TESTS_CONFIGMOUNTPOINT=/.config/wathola \
+  E2E_UPGRADE_TESTS_INTERVAL="${eventing_prober_interval}" \
+  go_test_e2e -tags=probe \
+    -timeout=30m \
+    ./test/upgrade \
+    --pipefile="${EVENTING_PROBER_FILE}" \
+    --readyfile="${EVENTING_READY_FILE}" \
+    --imagetemplate="${image_template}" &
+  eventing_prober_pid=$!
+
+  logger.debug "Eventing prober PID is ${eventing_prober_pid}"
+
+  echo ${eventing_prober_pid} > "${pid_file}"
+}
+
+function wait_for_eventing_prober_ready {
+  wait_for_file "${EVENTING_READY_FILE}" || return $?
+
+  logger.success 'Eventing prober is ready'
+}
+
+function end_eventing_prober {
+  local prober_pid
+  prober_pid="${1:?Pass a prober pid as arg[1]}"
+
+  end_prober 'Eventing' "${prober_pid}" "${EVENTING_PROBER_FILE}" || return $?
+}
+
+function check_eventing_upgraded {
+  local latest_version
+  latest_version="${1:?Pass a target eventing version as arg[1]}"
+
+  logger.debug 'Check KnativeEventing has the latest version with Ready status'
+  timeout 300 "[[ ! ( \$(oc get knativeeventing.operator.knative.dev \
+    knative-eventing -n ${EVENTING_NAMESPACE} -o=jsonpath='{.status.version}') \
+    == ${latest_version} && \$(oc get knativeeventing.operator.knative.dev \
+    knative-eventing -n ${EVENTING_NAMESPACE} \
+    -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') == True ) ]]" \
+    || return $?
+}
+
+function run_eventing_postupgrade_test {
+  logger.info 'Running Eventing post upgrade tests'
+  local image_template
+
+  cd "${KNATIVE_EVENTING_HOME}" || return $?
+
+  # FIXME: SRVKE-606 use registry.svc.ci.openshift.org image
+  image_template="quay.io/openshift-knative/{{.Name}}:${KNATIVE_EVENTING_VERSION}"
+
+  go_test_e2e -tags=postupgrade \
+    -timeout=10m ./test/upgrade \
+    --imagetemplate="${image_template}" \
+    || return $?
+
+  logger.success 'Eventing post upgrade tests passed'
 }