Feature: upgrade & downgrade tests

go.mod

@@ -15,17 +15,22 @@ require (
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/mailru/easyjson v0.7.1-0.20191009090205-6c0755d89d1e // indirect
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/openzipkin/zipkin-go v0.2.2
+	github.com/pelletier/go-toml v1.8.0
+	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rogpeppe/fastuuid v1.2.0
 	github.com/rogpeppe/go-internal v1.5.0 // indirect
 	github.com/stretchr/testify v1.5.1
 	github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible
+	github.com/wavesoftware/go-ensure v1.0.0
 	go.opencensus.io v0.22.3
 	go.opentelemetry.io/otel v0.2.3
 	go.uber.org/atomic v1.6.0
 	go.uber.org/zap v1.14.1
+	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 	gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect
 	google.golang.org/grpc v1.28.0
 	k8s.io/api v0.17.3

go.sum

@@ -763,8 +763,12 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
 github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
+github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
+github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
@@ -910,6 +914,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo=
 github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0=
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
+github.com/wavesoftware/go-ensure v1.0.0 h1:6X3gQL5psBWwtu/H9a+69xQ+JGTUELaLhgOB/iB3AQk=
+github.com/wavesoftware/go-ensure v1.0.0/go.mod h1:K2UAFSwMTvpiRGay/M3aEYYuurcR8S4A6HkQlJPV8k4=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
@@ -1353,6 +1359,8 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 helm.sh/helm/v3 v3.1.1/go.mod h1:WYsFJuMASa/4XUqLyv54s0U/f3mlAaRErGmyy4z921g=

hack/update-deps.sh

@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-readonly ROOT_DIR=$(dirname $0)/..
-source ${ROOT_DIR}/vendor/knative.dev/test-infra/scripts/library.sh
+readonly ROOT_DIR=$(dirname "$0")/..
+source "${ROOT_DIR}/vendor/knative.dev/test-infra/scripts/library.sh"
 
 set -o errexit
 set -o nounset
@@ -26,7 +26,7 @@ export GO111MODULE=on
 # This controls the release branch we track.
 VERSION="master"
 
-cd ${ROOT_DIR}
+cd "${ROOT_DIR}"
 
 # The list of dependencies that we track at HEAD and periodically
 # float forward in this repository.
@@ -55,8 +55,8 @@ fi
 go mod tidy
 go mod vendor
 
-rm -rf $(find vendor/ -name 'OWNERS')
-rm -rf $(find vendor/ -name '*_test.go')
+find vendor/ -name 'OWNERS' -delete
+find vendor/ -name '*_test.go'-delete
 
 export GOFLAGS=-mod=vendor
 

test/e2e-common.sh

@@ -21,7 +21,7 @@ export GO111MODULE=on
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh
 
 # If gcloud is not available make it a no-op, not an error.
-which gcloud &> /dev/null || gcloud() { echo "[ignore-gcloud $*]" 1>&2; }
+which gcloud &>/dev/null || gcloud() { echo "[ignore-gcloud $*]" 1>&2; }
 
 # Use GNU tools on macOS. Requires the 'grep' and 'gnu-sed' Homebrew formulae.
 if [ "$(uname)" == "Darwin" ]; then
@@ -43,19 +43,67 @@ readonly MT_CHANNEL_BASED_BROKER_DEFAULT_CONFIG="test/config/mt-channel-broker.y
 # Channel Based Broker Controller.
 readonly CHANNEL_BASED_BROKER_CONTROLLER="config/brokers/channel-broker"
 
-# Setup the Knative environment for running tests. This installs
-# Everything from the config dir but then removes the Channel Based Broker.
-# TODO: This should only install the core.
+# Should deploy a Knative Monitoring as well
+readonly DEPLOY_KNATIVE_MONITORING="${DEPLOY_KNATIVE_MONITORING:-1}"
+
+# Latest release. If user does not supply this as a flag, the latest
+# tagged release on the current branch will be used.
+readonly LATEST_RELEASE_VERSION=$(git describe --match "v[0-9]*" --abbrev=0)
+
+UNINSTALL_LIST=()
+
+# Setup the Knative environment for running tests.
 function knative_setup() {
-  # Install the latest Knative/eventing in the current cluster.
-  echo ">> Starting Knative Eventing"
-  echo "Installing Knative Eventing"
-  ko apply --strict -f ${EVENTING_CONFIG} || return 1
+  install_knative_eventing
+}
+
+# This installs everything from the config dir but then removes the Channel Based Broker.
+# TODO: This should only install the core.
+# Args:
+#  - $1 - if passed, it will be used as eventing config directory
+function install_knative_eventing() {
+  local kne_config
+  kne_config="${1:-${EVENTING_CONFIG}}"
+  # Install Knative Eventing in the current cluster.
+  echo "Installing Knative Eventing from: ${kne_config}"
+  if [ -f "${kne_config}" ] || [ -d "${kne_config}" ]; then
+    ko apply --strict -f "${kne_config}" || return $?
+  else
+    kubectl apply -f "${kne_config}" || return $?
+    UNINSTALL_LIST+=( "${kne_config}" )
+  fi
 
   wait_until_pods_running knative-eventing || fail_test "Knative Eventing did not come up"
 
-  echo "Installing Knative Monitoring"
-  start_knative_monitoring "${KNATIVE_MONITORING_RELEASE}" || fail_test "Knative Monitoring did not come up"
+  if ! (( DEPLOY_KNATIVE_MONITORING )); then return 0; fi
+
+  # Ensure knative monitoring is installed only once
+  knative_monitoring_pods=$(kubectl get pods -n knative-monitoring \
+    --field-selector status.phase=Running 2> /dev/null | tail -n +2 | wc -l)
+  if ! [[ ${knative_monitoring_pods} -gt 0 ]]; then
+    echo ">> Installing Knative Monitoring"
+    start_knative_monitoring "${KNATIVE_MONITORING_RELEASE}" || fail_test "Knative Monitoring did not come up"
+    UNINSTALL_LIST+=( "${KNATIVE_MONITORING_RELEASE}" )
+  else
+    echo ">> Knative Monitoring seems to be running, pods running: ${knative_monitoring_pods}."
+  fi
+}
+
+function install_head {
+  # Install Knative Eventing from HEAD in the current cluster.
+  echo ">> Installing Knative Eventing from HEAD"
+  install_knative_eventing || \
+    fail_test "Knative HEAD installation failed"
+}
+
+function install_latest_release() {
+  header ">> Installing Knative Eventing latest public release"
+  local url="https://github.com/knative/eventing/releases/download/${LATEST_RELEASE_VERSION}"
+  local yaml="eventing.yaml"
+
+  install_knative_eventing \
+    "${url}/${yaml}" || \
+    fail_test "Knative latest release installation failed"
 }
 
 function install_broker() {
@@ -81,6 +129,14 @@ function knative_teardown() {
   echo "Uninstalling Knative Eventing"
   ko delete --ignore-not-found=true --now --timeout 60s -f ${EVENTING_CONFIG}
   wait_until_object_does_not_exist namespaces knative-eventing
+
+  echo ">> Uninstalling dependencies"
+  for i in ${!UNINSTALL_LIST[@]}; do
+    # We uninstall elements in the reverse of the order they were installed.
+    local YAML="${UNINSTALL_LIST[$(( ${#array[@]} - $i ))]}"
+    echo ">> Bringing down YAML: ${YAML}"
+    kubectl delete --ignore-not-found=true -f "${YAML}" || return 1
+  done
 }
 
 # Add function call to trap
@@ -102,20 +158,20 @@ function test_setup() {
   echo ">> Setting up logging..."
 
   # Install kail if needed.
-  if ! which kail > /dev/null; then
-    bash <( curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$GOPATH/bin"
+  if ! which kail >/dev/null; then
+    bash <(curl -sfL https://raw.githubusercontent.com/boz/kail/master/godownloader.sh) -b "$GOPATH/bin"
   fi
 
   # Capture all logs.
-  kail > ${ARTIFACTS}/k8s.log.txt &
+  kail >${ARTIFACTS}/k8s.log.txt &
   local kail_pid=$!
   # Clean up kail so it doesn't interfere with job shutting down
   add_trap "kill $kail_pid || true" EXIT
 
   install_test_resources || return 1
 
-  # Publish test images.
-  $(dirname $0)/upload-test-images.sh e2e || fail_test "Error uploading test images"
+  echo ">> Publish test images"
+  "$(dirname "$0")/upload-test-images.sh" e2e || fail_test "Error uploading test images"
 }
 
 # Tear down resources used in the eventing tests.
@@ -146,7 +202,7 @@ function dump_extra_cluster_state() {
   # Collecting logs from all knative's eventing pods.
   echo "============================================================"
   local namespace="knative-eventing"
-  for pod in $(kubectl get pod -n $namespace | grep Running | awk '{print $1}' ); do
+  for pod in $(kubectl get pod -n $namespace | grep Running | awk '{print $1}'); do
     for container in $(kubectl get pod "${pod}" -n $namespace -ojsonpath='{.spec.containers[*].name}'); do
       echo "Namespace, Pod, Container: ${namespace}, ${pod}, ${container}"
       kubectl logs -n $namespace "${pod}" -c "${container}" || true
@@ -157,3 +213,26 @@ function dump_extra_cluster_state() {
     done
   done
 }
+
+function wait_for_file() {
+  local file timeout waits
+  file="$1"
+  waits=300
+  timeout=$waits
+
+  echo "Waiting for existance of file: ${file}"
+
+  while [ ! -f "${file}" ]; do
+    # When the timeout is equal to zero, show an error and leave the loop.
+    if [ "${timeout}" == 0 ]; then
+      echo "ERROR: Timeout (${waits}s) while waiting for the file ${file}."
+      return 1
+    fi
+
+    sleep 1
+
+    # Decrease the timeout of one
+    ((timeout--))
+  done
+  return 0
+}

test/e2e-tests.sh

@@ -26,7 +26,7 @@
 
 export GO111MODULE=on
 
-source $(dirname $0)/e2e-common.sh
+source "$(dirname "$0")/e2e-common.sh"
 
 # Script entry point.
 

test/e2e-upgrade-tests.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Docs -> file://./upgrade/README.md
+
+# Script entry point.
+
+export GO111MODULE=on
+
+source "$(dirname "$0")/e2e-common.sh"
+
+# Overrides
+
+function knative_setup {
+  install_latest_release || fail_test 'Installing latest release of Knative Eventing failed'
+}
+
+function install_test_resources {
+  # Nothing to install before tests
+  true
+}
+
+function uninstall_test_resources {
+  # Nothing to uninstall after tests
+  true
+}
+
+initialize $@ --skip-istio-addon
+
+TIMEOUT=${TIMEOUT:-30m}
+
+header "Running preupgrade tests"
+
+go_test_e2e -tags=preupgrade -timeout="${TIMEOUT}" ./test/upgrade || fail_test
+
+header "Starting prober test"
+rm -fv /tmp/prober-ready
+go_test_e2e -tags=probe -timeout="${TIMEOUT}" ./test/upgrade &
+PROBER_PID=$!
+echo "Prober PID is ${PROBER_PID}"
+
+wait_for_file /tmp/prober-ready || fail_test
+
+header "Performing upgrade to HEAD"
+install_head || fail_test 'Installing HEAD version of eventing failed'
+install_channel_crds || fail_test 'Installing HEAD channel CRDs failed'
+install_broker || fail_test 'Installing HEAD Broker failed'
+
+header "Running postupgrade tests"
+go_test_e2e -tags=postupgrade -timeout="${TIMEOUT}" ./test/upgrade || fail_test
+
+header "Performing downgrade to latest release"
+install_latest_release || fail_test 'Installing latest release of Knative Eventing failed'
+
+header "Running postdowngrade tests"
+go_test_e2e -tags=postdowngrade -timeout="${TIMEOUT}" ./test/upgrade || fail_test
+
+# The prober is blocking on /tmp/prober-signal to know when it should exit.
+echo "done" > /tmp/prober-signal
+
+header "Waiting for prober test"
+wait ${PROBER_PID} || fail_test "Prober failed"
+
+success

test/e2e/channel_single_event_test.go

@@ -37,7 +37,7 @@ func TestSingleBinaryEventForChannel(t *testing.T) {
 	helpers.SingleEventForChannelTestHelper(
 		t,
 		cloudevents.Binary,
-		"v1alpha1",
+		helpers.SubscriptionV1alpha1,
 		"",
 		channelTestRunner,
 	)
@@ -47,7 +47,7 @@ func TestSingleStructuredEventForChannel(t *testing.T) {
 	helpers.SingleEventForChannelTestHelper(
 		t,
 		cloudevents.Structured,
-		"v1alpha1",
+		helpers.SubscriptionV1alpha1,
 		"",
 		channelTestRunner,
 	)
@@ -57,7 +57,7 @@ func TestSingleBinaryEventForChannelV1Beta1(t *testing.T) {
 	helpers.SingleEventForChannelTestHelper(
 		t,
 		cloudevents.Binary,
-		"v1beta1",
+		helpers.SubscriptionV1beta1,
 		"",
 		channelTestRunner,
 	)
@@ -67,7 +67,7 @@ func TestSingleBinaryEventForChannelV1Beta1SubscribeToV1Alpha1(t *testing.T) {
 	helpers.SingleEventForChannelTestHelper(
 		t,
 		cloudevents.Binary,
-		"v1beta1",
+		helpers.SubscriptionV1beta1,
 		"messaging.knative.dev/v1alpha1",
 		channelTestRunner,
 	)
@@ -77,7 +77,7 @@ func TestSingleStructuredEventForChannelV1Beta1(t *testing.T) {
 	helpers.SingleEventForChannelTestHelper(
 		t,
 		cloudevents.Structured,
-		"v1beta1",
+		helpers.SubscriptionV1beta1,
 		"",
 		channelTestRunner,
 	)