From 28edfb1d9bf5bf488e1385d2ddc81af192f9e709 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chris=20Suszy=C5=84ski?= <ksuszyns@redhat.com>
Date: Fri, 6 Dec 2019 22:21:39 +0100
Subject: [PATCH] Invocation of upstream knative/serving-operator e2e tests

A bash version of test invocation of upstream knative/serving-operator.

There is a patch that skips TestKnativeServingDeployment/configure due
to SRVKS-241.

There is a 0.9.0 version of knative/serving-operator executed, as
current serverless operator do not handle
knativeservings.operator.knative.dev yet. After supporting that
versions should be set to support/v0.10.0. For 0.11 we can use use
knative instead of fork (cardil) as master has configurable test
namespace.
---
 hack/lib/serverless.bash                      | 41 ++++----
 test/e2e-tests.sh                             | 17 ++--
 test/lib.bash                                 | 93 ++++++++++++++++---
 ...tive-serving-operator-skip-configure.patch | 14 +++
 4 files changed, 129 insertions(+), 36 deletions(-)
 create mode 100644 test/patches/SRVKS-241-knative-serving-operator-skip-configure.patch

diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash
index 9f9fdca8ee..b7e0a8c171 100644
--- a/hack/lib/serverless.bash
+++ b/hack/lib/serverless.bash
@@ -10,37 +10,45 @@ function ensure_serverless_installed {
 }
 
 function install_serverless_previous {
-  local rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
+  local rootdir current_csv previous_csv
+  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
 
   # Remove installplan from previous installations, leaving this would make the operator
   # upgrade to the latest version immediately
-  local current_csv=$(${rootdir}/hack/catalog.sh | grep currentCSV | awk '{ print $2 }')
-  remove_installplan $current_csv
+  current_csv=$("${rootdir}/hack/catalog.sh" | grep currentCSV | awk '{ print $2 }')
+  remove_installplan "$current_csv"
 
-  local previous_csv=$(${rootdir}/hack/catalog.sh | grep replaces: | tail -n1 | awk '{ print $2 }')
-  deploy_serverless_operator $previous_csv  || return $?
+  previous_csv=$("${rootdir}/hack/catalog.sh" | grep replaces: | tail -n1 | awk '{ print $2 }')
+  deploy_serverless_operator "$previous_csv"  || return $?
   deploy_knativeserving_cr || return $?
 }
 
 function remove_installplan {
-  local install_plan=$(find_install_plan $1)
+  local install_plan
+  install_plan=$(find_install_plan $1)
   if [[ -n $install_plan ]]; then
-    oc delete $install_plan -n ${OPERATORS_NAMESPACE}
+    oc delete "$install_plan" -n "${OPERATORS_NAMESPACE}"
   fi
 }
 
 function install_serverless_latest {
-  local rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
+  deploy_serverless_operator_latest || return $?
+  deploy_knativeserving_cr || return $?
+}
+
+function deploy_serverless_operator_latest {
+  local rootdir csv
+  rootdir="$(dirname "$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")")"
   # Get the current/latest CSV
-  local csv=$(${rootdir}/hack/catalog.sh | grep currentCSV | awk '{ print $2 }')
+  csv=$("${rootdir}/hack/catalog.sh" | grep currentCSV | awk '{ print $2 }')
 
-  deploy_serverless_operator $csv || return $?
-  deploy_knativeserving_cr || return $?
+  deploy_serverless_operator "${csv}"
 }
 
 function deploy_serverless_operator {
   logger.info 'Install the Serverless Operator'
-  local csv=$1
+  local csv
+  csv="$1"
 
   cat <<EOF | oc apply -f - || return $?
 apiVersion: operators.coreos.com/v1alpha1
@@ -58,16 +66,17 @@ spec:
 EOF
 
   # Approve the initial installplan automatically
-  approve_csv $csv || return 5
+  approve_csv "$csv" || return 5
 }
 
 function approve_csv {
-  local csv_version=$1
+  local csv_version install_plan
+  csv_version=$1
 
   # Wait for the installplan to be available
   timeout 900 "[[ -z \$(find_install_plan $csv_version) ]]" || return 1
 
-  local install_plan=$(find_install_plan $csv_version)
+  install_plan=$(find_install_plan $csv_version)
   oc get $install_plan -n ${OPERATORS_NAMESPACE} -o yaml | sed 's/\(.*approved:\) false/\1 true/' | oc replace -f -
 
   timeout 300 "[[ \$(oc get ClusterServiceVersion $csv_version -n ${OPERATORS_NAMESPACE} -o jsonpath='{.status.phase}') != Succeeded ]]" || return 1
@@ -86,7 +95,7 @@ function deploy_knativeserving_cr {
   logger.info 'Deploy Knative Serving'
 
   # Wait for the CRD to appear
-  timeout 900 '[[ $(oc get crd | grep -c knativeservings) -eq 0 ]]' || return 6
+  timeout 900 "[[ \$(oc get crd | grep -c knativeservings) -eq 0 ]]" || return 6
 
   # Install Knative Serving
   cat <<EOF | oc apply -f - || return $?
diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh
index e169ccb20f..fd919ae94a 100755
--- a/test/e2e-tests.sh
+++ b/test/e2e-tests.sh
@@ -19,13 +19,18 @@ failed=0
 # Run serverless-operator specific tests.
 (( !failed )) && run_e2e_tests || failed=4
 
-# Run upstream knative serving tests
-RUN_KNATIVE_SERVING_UPGRADE_TESTS=false
-RUN_KNATIVE_SERVING_E2E=true
+RUN_KNATIVE_SERVING_UPGRADE_TESTS=${RUN_KNATIVE_SERVING_UPGRADE_TESTS:-false}
+RUN_KNATIVE_SERVING_E2E=${RUN_KNATIVE_SERVING_E2E:-true}
+export RUN_KNATIVE_SERVING_UPGRADE_TESTS RUN_KNATIVE_SERVING_E2E
+
+# Run upstream knative serving operator tests
+(( !failed )) && deploy_serverless_operator_latest || failed=11
+(( !failed )) && run_knative_serving_operator_tests 'support/v0.9.0' 'cardil' || failed=12
 
-(( !failed )) && ensure_serverless_installed || failed=5
-(( !failed )) && run_knative_serving_tests "v0.10.0" || failed=6
-(( !failed )) && teardown_serverless || failed=7
+# Run upstream knative serving tests
+(( !failed )) && ensure_serverless_installed || failed=6
+(( !failed )) && run_knative_serving_tests "v0.10.0" || failed=7
+(( !failed )) && teardown_serverless || failed=8
 
 (( failed )) && dump_state
 (( failed )) && exit $failed
diff --git a/test/lib.bash b/test/lib.bash
index c41847ed5c..7517b52366 100644
--- a/test/lib.bash
+++ b/test/lib.bash
@@ -45,23 +45,30 @@ function run_e2e_tests {
     && return 1
 }
 
-function run_knative_serving_tests {
-  (
-  local knative_version=$1
-  # Setup a temporary GOPATH to safely check out the repository without breaking other things.
+# Setup a temporary GOPATH to safely check out the repository without breaking other things.
+# CAUTION: function overrides GOPATH so use it in subshell or restore original value!
+function make_temporary_gopath {
   local tmp_gopath
   tmp_gopath="$(mktemp -d -t gopath-XXXXXXXXXX)"
-  cp -r "$GOPATH/bin" "$tmp_gopath"
+  cp -rv "$(go env GOPATH)/bin" "${tmp_gopath}"
+  logger.info "Temporary GOPATH is: ${tmp_gopath}"
   export GOPATH="$tmp_gopath"
+}
+
+function run_knative_serving_tests {
+  (
+  local knative_version=$1
+  make_temporary_gopath
 
   # Save the rootdir before changing dir
   rootdir="$(dirname "$(dirname "$(realpath "${BASH_SOURCE[0]}")")")"
 
   # Checkout the relevant code to run
   mkdir -p "$GOPATH/src/knative.dev"
-  cd "$GOPATH/src/knative.dev" || return $?
-  git clone -b "release-${knative_version}" --single-branch https://github.com/openshift/knative-serving.git serving
-  cd serving || return $?
+  git clone --branch "release-${knative_version}" \
+    --depth=1 https://github.com/openshift/knative-serving.git \
+    "${GOPATH}/src/knative.dev/serving"
+  pushd "${GOPATH}/src/knative.dev/serving" || return $?
 
   # Remove unneeded manifest
   rm test/config/100-istio-default-domain.yaml
@@ -77,6 +84,8 @@ function run_knative_serving_tests {
   image_template="registry.svc.ci.openshift.org/openshift/knative-${knative_version}:knative-serving-test-{{.Name}}"
   export GATEWAY_NAMESPACE_OVERRIDE="knative-serving-ingress"
 
+  git describe --always --tags --dirty
+
   # Rolling upgrade tests must run first because they upgrade Serverless to the latest version
   if [[ $RUN_KNATIVE_SERVING_UPGRADE_TESTS == true ]]; then
      run_knative_serving_rolling_upgrade_tests || failed=1
@@ -86,7 +95,8 @@ function run_knative_serving_tests {
     run_knative_serving_e2e_and_conformance_tests || failed=1
   fi
 
-  rm -rf "$tmp_gopath"
+  rm -rf "${GOPATH}"
+  popd || return $?
   return $failed
   )
 }
@@ -170,6 +180,51 @@ function end_prober_test {
   wait ${PROBER_PID}
 }
 
+function run_knative_serving_operator_tests {
+  (
+  local version target serverless_rootdir exitstatus patchfile fork gitdesc
+  version="$1"
+  fork="${2:-knative}"
+  serverless_rootdir="$(pwd)"
+  make_temporary_gopath
+
+  logger.info "Checkout the code ${fork}/serving-operator @ ${version}"
+  mkdir -p "${GOPATH}/src/knative.dev"
+  target="${GOPATH}/src/knative.dev/serving-operator"
+  git clone --branch "${version}" --depth 1 \
+    "https://github.com/${fork}/serving-operator.git" \
+    "${target}"
+  pushd "${target}" || return $?
+
+  patchfile="${serverless_rootdir}/test/patches/SRVKS-241-knative-serving-operator-skip-configure.patch"
+  logger.info "Apply walkaround for SRVKS-241"
+  logger.debug "Patchfile is: ${patchfile}"
+  [ -f "${patchfile}" ] || return $?
+  patch --strip=0 < "${patchfile}" || return $?
+
+  gitdesc=$(git describe --always --tags --dirty)
+
+  exitstatus=0
+
+  logger.info "Run tests of knative/serving-operator @ ${gitdesc}"
+  env TEST_NAMESPACE='knative-serving' \
+    go test -v -tags=e2e -count=1 -timeout=30m -parallel=1 ./test/e2e \
+      --kubeconfig "$KUBECONFIG" \
+    || exitstatus=5$? && true
+
+  if (( !exitstatus )); then
+    logger.success 'Tests have passed'
+  else
+    logger.error 'Tests have failures!'
+  fi
+
+  logger.info "Removing GOPATH: ${GOPATH}"
+  rm -rf "${GOPATH}"
+  popd || return $?
+  return $exitstatus
+  )
+}
+
 function teardown {
   if [ -n "$OPENSHIFT_BUILD_NAMESPACE" ]; then
     logger.warn 'Skipping teardown as we are running on Openshift CI'
@@ -194,23 +249,33 @@ function dump_state {
   dump_cluster_state
   dump_openshift_olm_state
   dump_openshift_ingress_state
+  dump_knative_state
 }
 
 function dump_openshift_olm_state {
   logger.info "Dump of subscriptions.operators.coreos.com"
   # This is for status checking.
-  oc get subscriptions.operators.coreos.com -o yaml --all-namespaces
+  oc get subscriptions.operators.coreos.com -o yaml --all-namespaces || true
   logger.info "Dump of catalog operator log"
-  oc logs -n openshift-operator-lifecycle-manager deployment/catalog-operator
+  oc logs -n openshift-operator-lifecycle-manager deployment/catalog-operator || true
 }
 
 function dump_openshift_ingress_state {
   logger.info "Dump of routes.route.openshift.io"
-  oc get routes.route.openshift.io -o yaml --all-namespaces
+  oc get routes.route.openshift.io -o yaml --all-namespaces || true
   logger.info "Dump of routes.serving.knative.dev"
-  oc get routes.serving.knative.dev -o yaml --all-namespaces
+  oc get routes.serving.knative.dev -o yaml --all-namespaces || true
   logger.info "Dump of openshift-ingress log"
-  oc logs deployment/knative-openshift-ingress -n "$SERVING_NAMESPACE"
+  oc logs deployment/knative-openshift-ingress -n "$SERVING_NAMESPACE" || true
+}
+
+function dump_knative_state {
+  logger.info 'Dump of knative state'
+  oc describe knativeserving knative-serving -n "$SERVING_NAMESPACE" || true
+  oc get smcp --all-namespaces || true
+  oc get smmr --all-namespaces || true
+  oc get pods -n "$SERVING_NAMESPACE" || true
+  oc get ksvc --all-namespaces || true
 }
 
 # == Test users
diff --git a/test/patches/SRVKS-241-knative-serving-operator-skip-configure.patch b/test/patches/SRVKS-241-knative-serving-operator-skip-configure.patch
new file mode 100644
index 0000000000..adba793b3b
--- /dev/null
+++ b/test/patches/SRVKS-241-knative-serving-operator-skip-configure.patch
@@ -0,0 +1,14 @@
+diff --git test/e2e/knativeservingdeployment_test.go test/e2e/knativeservingdeployment_test.go
+index ed985a1..20deb22 100644
+--- test/e2e/knativeservingdeployment_test.go
++++ test/e2e/knativeservingdeployment_test.go
+@@ -58,8 +58,7 @@ func TestKnativeServingDeployment(t *testing.T) {
+ 	})
+ 
+ 	t.Run("configure", func(t *testing.T) {
+-		knativeServingVerify(t, clients, names)
+-		knativeServingConfigure(t, clients, names)
++		t.Skip("Skip due to SRVKS-241")
+ 	})
+ 
+ 	// Delete the deployments one by one to see if they will be recreated.