Add upgrade tests into the e2e-tests-latest-eventing

pkg/reconciler/knativeeventing/knativeeventing.go

@@ -214,8 +214,7 @@ func (r *Reconciler) updateStatus(desired *eventingv1alpha1.KnativeEventing) (*e
 // Delete obsolete resources from previous versions
 func (r *Reconciler) deleteObsoleteResources(manifest *mf.Manifest, instance *eventingv1alpha1.KnativeEventing) error {
 	resource := &unstructured.Unstructured{}
-
-	resource.SetNamespace("knative-eventing")
+	resource.SetNamespace(instance.GetNamespace())
 
 	// Remove old resources from 0.12
 	// https://github.com/knative/eventing-operator/issues/90

test/e2e/e2e.go → test/client/setup.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Knative Authors
+Copyright 2020 The Knative Authors
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
@@ -11,7 +11,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package client
 
 import (
 	"testing"
@@ -20,6 +20,7 @@ import (
 	// Apparently just importing it is enough. @_@ side effects @_@.
 	// https://github.com/kubernetes/client-go/issues/242
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+
 	"knative.dev/eventing-operator/test"
 	pkgTest "knative.dev/pkg/test"
 )

test/e2e-upgrade-tests.sh

@@ -28,6 +28,9 @@
 
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh
 
+# Latest eventing operator release.
+readonly LATEST_EVENTING_RELEASE_VERSION=$(git describe --match "v[0-9]*" --abbrev=0)
+
 OPERATOR_DIR=$(dirname $0)/..
 KNATIVE_EVENTING_DIR=${OPERATOR_DIR}/..
 
@@ -48,9 +51,37 @@ function install_eventing_operator() {
   wait_until_pods_running default || fail_test "Eventing Operator did not come up"
 }
 
+function install_latest_operator_release() {
+  header "Installing Knative Eventing operator latest public release"
+  local full_url="https://github.com/knative/eventing-operator/releases/download/${LATEST_EVENTING_RELEASE_VERSION}/eventing-operator.yaml"
+
+  local release_yaml="$(mktemp)"
+  wget "${full_url}" -O "${release_yaml}" \
+      || fail_test "Unable to download latest Knative Eventing Operator release."
+
+  kubectl apply -f "${release_yaml}" || fail_test "Knative Eventing Operator latest release installation failed"
+  create_custom_resource
+  wait_until_pods_running ${TEST_NAMESPACE}
+}
+
+function create_custom_resource() {
+  echo ">> Creating the custom resource of Knative Eventing:"
+  cat <<EOF | kubectl apply -f -
+apiVersion: operator.knative.dev/v1alpha1
+kind: KnativeEventing
+metadata:
+  name: knative-eventing
+  namespace: ${TEST_NAMESPACE}
+EOF
+}
+
 function knative_setup() {
   echo ">> Creating test namespaces"
   kubectl create namespace $TEST_NAMESPACE
+  install_latest_operator_release
+}
+
+function install_head() {
   generate_latest_eventing_manifest
   install_eventing_operator
 }
@@ -95,12 +126,16 @@ function generate_latest_eventing_manifest() {
 # Skip installing istio as an add-on
 initialize $@ --skip-istio-addon
 
+TIMEOUT=20m
+
+install_head
+
 # If we got this far, the operator installed Knative Eventing
 header "Running tests for Knative Eventing Operator"
 failed=0
 
-# Run the integration tests
-go_test_e2e -timeout=20m ./test/e2e || failed=1
+# Run the postupgrade tests
+go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade || failed=1
 
 # Require that tests succeeded.
 (( failed )) && fail_test

test/e2e/knativeeventingdeployment_test.go

@@ -1,7 +1,7 @@
 // +build e2e
 
 /*
-Copyright 2019 The Knative Authors
+Copyright 2020 The Knative Authors
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
@@ -16,18 +16,10 @@ limitations under the License.
 package e2e
 
 import (
-	"errors"
-	"path/filepath"
-	"runtime"
 	"testing"
 
-	"k8s.io/apimachinery/pkg/api/meta"
-
-	mf "github.com/manifestival/manifestival"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"knative.dev/eventing-operator/test"
+	"knative.dev/eventing-operator/test/client"
 	"knative.dev/eventing-operator/test/resources"
 	"knative.dev/pkg/test/logstream"
 )
@@ -36,7 +28,7 @@ import (
 func TestKnativeEventingDeployment(t *testing.T) {
 	cancel := logstream.Start(t)
 	defer cancel()
-	clients := Setup(t)
+	clients := client.Setup(t)
 
 	names := test.ResourceNames{
 		KnativeEventing: test.EventingOperatorName,
@@ -47,119 +39,24 @@ func TestKnativeEventingDeployment(t *testing.T) {
 	defer test.TearDown(clients, names)
 
 	// Create a KnativeEventing
-	if _, err := resources.CreateKnativeEventing(clients.KnativeEventing(), names); err != nil {
+	if _, err := resources.EnsureKnativeEventingExists(clients.KnativeEventing(), names); err != nil {
 		t.Fatalf("KnativeService %q failed to create: %v", names.KnativeEventing, err)
 	}
 
 	// Test if KnativeEventing can reach the READY status
 	t.Run("create", func(t *testing.T) {
-		knativeEventingVerify(t, clients, names)
+		resources.AssertKEOperatorCRReadyStatus(t, clients, names)
 	})
 
 	// Delete the deployments one by one to see if they will be recreated.
 	t.Run("restore", func(t *testing.T) {
-		knativeEventingVerify(t, clients, names)
-		deploymentRecreation(t, clients, names)
+		resources.AssertKEOperatorCRReadyStatus(t, clients, names)
+		resources.DeleteAndVerifyDeployments(t, clients, names)
 	})
 
 	// Delete the KnativeEventing to see if all resources will be removed
 	t.Run("delete", func(t *testing.T) {
-		knativeEventingVerify(t, clients, names)
-		knativeEventingDelete(t, clients, names)
-	})
-}
-
-// knativeEventingVerify verifies if the KnativeEventing can reach the READY status.
-func knativeEventingVerify(t *testing.T, clients *test.Clients, names test.ResourceNames) {
-	if _, err := resources.WaitForKnativeEventingState(clients.KnativeEventing(), names.KnativeEventing,
-		resources.IsKnativeEventingReady); err != nil {
-		t.Fatalf("KnativeService %q failed to get to the READY status: %v", names.KnativeEventing, err)
-	}
-
-}
-
-// deploymentRecreation verify whether all the deployments for knative eventing are able to recreate, when they are deleted.
-func deploymentRecreation(t *testing.T, clients *test.Clients, names test.ResourceNames) {
-	dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{})
-	if err != nil {
-		t.Fatalf("Failed to get any deployment under the namespace %q: %v",
-			test.EventingOperatorNamespace, err)
-	}
-	if len(dpList.Items) == 0 {
-		t.Fatalf("No deployment under the namespace %q was found",
-			test.EventingOperatorNamespace)
-	}
-	// Delete the first deployment and verify the operator recreates it
-	deployment := dpList.Items[0]
-	if err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Delete(deployment.Name,
-		&metav1.DeleteOptions{}); err != nil {
-		t.Fatalf("Failed to delete deployment %s/%s: %v", deployment.Namespace, deployment.Name, err)
-	}
-
-	waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
-		dep, err := clients.KubeClient.Kube.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{})
-		if err != nil {
-			// If the deployment is not found, we continue to wait for the availability.
-			if apierrs.IsNotFound(err) {
-				return false, nil
-			}
-			return false, err
-		}
-		return resources.IsDeploymentAvailable(dep)
+		resources.AssertKEOperatorCRReadyStatus(t, clients, names)
+		resources.KEOperatorCRDelete(t, clients, names)
 	})
-
-	if waitErr != nil {
-		t.Fatalf("The deployment %s/%s failed to reach the desired state: %v", deployment.Namespace, deployment.Name, err)
-	}
-
-	if _, err := resources.WaitForKnativeEventingState(clients.KnativeEventing(), test.EventingOperatorName,
-		resources.IsKnativeEventingReady); err != nil {
-		t.Fatalf("KnativeService %q failed to reach the desired state: %v", test.EventingOperatorName, err)
-	}
-	t.Logf("The deployment %s/%s reached the desired state.", deployment.Namespace, deployment.Name)
-}
-
-// knativeEventingDelete deletes tha KnativeEventing to see if all resources will be deleted
-func knativeEventingDelete(t *testing.T, clients *test.Clients, names test.ResourceNames) {
-	if err := clients.KnativeEventing().Delete(names.KnativeEventing, &metav1.DeleteOptions{}); err != nil {
-		t.Fatalf("KnativeEventing %q failed to delete: %v", names.KnativeEventing, err)
-	}
-	_, b, _, _ := runtime.Caller(0)
-	m, err := mf.NewManifest(filepath.Join((filepath.Dir(b)+"/.."), "config/"), false, clients.Config)
-	if err != nil {
-		t.Fatal("Failed to load manifest", err)
-	}
-	if err := verifyNoKnativeEventings(clients); err != nil {
-		t.Fatal(err)
-	}
-	for _, u := range m.Resources {
-		if u.GetKind() == "Namespace" {
-			// The namespace should be skipped, because when the CR is removed, the Manifest to be removed has
-			// been modified, since the namespace can be injected.
-			continue
-		}
-		waitErr := wait.PollImmediate(resources.Interval, resources.Timeout, func() (bool, error) {
-			gvrs, _ := meta.UnsafeGuessKindToResource(u.GroupVersionKind())
-			if _, err := clients.Dynamic.Resource(gvrs).Get(u.GetName(), metav1.GetOptions{}); apierrs.IsNotFound(err) {
-				return true, nil
-			}
-			return false, err
-		})
-
-		if waitErr != nil {
-			t.Fatalf("The %s %s failed to be deleted: %v", u.GetKind(), u.GetName(), waitErr)
-		}
-		t.Logf("The %s %s has been deleted.", u.GetKind(), u.GetName())
-	}
-}
-
-func verifyNoKnativeEventings(clients *test.Clients) error {
-	eventings, err := clients.KnativeEventingAll().List(metav1.ListOptions{})
-	if err != nil {
-		return err
-	}
-	if len(eventings.Items) > 0 {
-		return errors.New("Unable to verify cluster-scoped resources are deleted if any KnativeEventing exists")
-	}
-	return nil
 }

test/resources/knativeeventing.go

@@ -26,12 +26,14 @@ import (
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	"knative.dev/pkg/test/logging"
 	"knative.dev/eventing-operator/pkg/apis/eventing/v1alpha1"
 	eventingv1alpha1 "knative.dev/eventing-operator/pkg/client/clientset/versioned/typed/eventing/v1alpha1"
 	"knative.dev/eventing-operator/test"
-	"knative.dev/pkg/test/logging"
 )
 
 const (
@@ -61,16 +63,20 @@ func WaitForKnativeEventingState(clients eventingv1alpha1.KnativeEventingInterfa
 	return lastState, nil
 }
 
-// CreateKnativeEventing creates a KnativeEventingServing with the name names.KnativeEventing under the namespace names.Namespace.
-func CreateKnativeEventing(clients eventingv1alpha1.KnativeEventingInterface, names test.ResourceNames) (*v1alpha1.KnativeEventing, error) {
-	ks := &v1alpha1.KnativeEventing{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      names.KnativeEventing,
-			Namespace: names.Namespace,
-		},
+// EnsureKnativeEventingExists creates a KnativeEventingServing with the name names.KnativeEventing under the namespace names.Namespace.
+func EnsureKnativeEventingExists(clients eventingv1alpha1.KnativeEventingInterface, names test.ResourceNames) (*v1alpha1.KnativeEventing, error) {
+	// If this function is called by the upgrade tests, we only create the custom resource, if it does not exist.
+	ke, err := clients.Get(names.KnativeEventing, metav1.GetOptions{})
+	if apierrs.IsNotFound(err) {
+		ke := &v1alpha1.KnativeEventing{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      names.KnativeEventing,
+				Namespace: names.Namespace,
+			},
+		}
+		return clients.Create(ke)
 	}
-	svc, err := clients.Create(ks)
-	return svc, err
+	return ke, err
 }
 
 // IsKnativeEventingReady will check the status conditions of the KnativeEventing and return true if the KnativeEventing is ready.