Suite for running upgrade tests in Knative components

test/upgrade/asserts_test.go

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020 The Knative Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package upgrade_test
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type assertions struct {
+	t *testing.T
+}
+
+func (a assertions) textContains(haystack string, needles texts) {
+	for _, needle := range needles.elms {
+		if !strings.Contains(haystack, needle) {
+			a.t.Errorf(
+				"expected \"%s\" is not in: `%s`",
+				needle, haystack,
+			)
+		}
+	}
+}
+
+func (a assertions) arraysEqual(actual []string, expected []string) {
+	if !reflect.DeepEqual(actual, expected) {
+		a.t.Errorf("arrays differ:\n  actual: %#v\nexpected: %#v", actual, expected)
+	}
+}

test/upgrade/execute_failures_test.go

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2020 The Knative Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package upgrade_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestSuiteExecuteWithFailures(t *testing.T) {
+	for i := 1; i <= 8; i++ {
+		for j := 1; j <= 2; j++ {
+			fp := failurePoint{
+				step:    i,
+				element: j,
+			}
+			testSuiteExecuteWithFailingStep(fp, t)
+		}
+	}
+}
+
+var allTestsFilter = func(_, _ string) (bool, error) { return true, nil }
+
+func testSuiteExecuteWithFailingStep(fp failurePoint, t *testing.T) {
+	assert := assertions{t: t}
+	testName := fmt.Sprintf("FailAt-%d-%d", fp.step, fp.element)
+	t.Run(testName, func(t *testing.T) {
+		var output string
+		suite := completeSuiteExample(fp)
+		txt := expectedTexts(suite, fp)
+		txt.append(upgradeTestRunning, upgradeTestFailure)
+		log, buf := newExampleZap()
+
+		it := []testing.InternalTest{{
+			Name: testName,
+			F: func(t *testing.T) {
+				c, _ := newConfig(t)
+				c.Log = log
+				suite.Execute(c)
+			},
+		}}
+		var ok bool
+		testOutput := captureStdOutput(func() {
+			ok = testing.RunTests(allTestsFilter, it)
+		})
+		output = buf.String()
+
+		if ok {
+			t.Fatal("didn't failed, but should")
+		}
+
+		assert.textContains(output, txt)
+		assert.textContains(testOutput, texts{
+			elms: []string{
+				fmt.Sprintf("--- FAIL: FailAt-%d-%d", fp.step, fp.element),
+			},
+		})
+	})
+}
+
+func captureStdOutput(call func()) string {
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	defer func() {
+		os.Stdout = rescueStdout
+	}()
+
+	call()
+
+	_ = w.Close()
+	out, _ := ioutil.ReadAll(r)
+	return string(out)
+}

test/upgrade/functions.go

@@ -0,0 +1,155 @@
+/*
+ * Copyright 2020 The Knative Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package upgrade
+
+import (
+	"time"
+
+	"go.uber.org/zap"
+)
+
+// Execute the Suite of upgrade tests with a Configuration given.
+func (s *Suite) Execute(c Configuration) {
+	l := c.logger()
+	se := suiteExecution{
+		suite:         enrichSuite(s),
+		configuration: c,
+		failed:        false,
+		logger:        l,
+	}
+	l.Info("🏃 Running upgrade test suite...")
+
+	se.execute()
+
+	if !se.failed {
+		l.Info("🥳🎉 Success! Upgrade suite completed without errors.")
+	} else {
+		l.Error("💣🤬💔️ Upgrade suite have failed!")
+	}
+}
+
+// NewOperation creates a new upgrade operation or test.
+func NewOperation(name string, handler func(c Context)) Operation {
+	return &simpleOperation{name: name, handler: handler}
+}
+
+// NewBackgroundVerification is convenience function to easily setup a
+// background operation that will setup environment and then verify environment
+// status after receiving a StopEvent.
+func NewBackgroundVerification(name string, setup func(c Context), verify func(c Context)) BackgroundOperation {
+	return NewBackgroundOperation(name, setup, func(bc BackgroundContext) {
+		WaitForStopEvent(bc, WaitForStopEventConfiguration{
+			Name: name,
+			OnStop: func(event StopEvent) {
+				verify(Context{
+					T:   event.T,
+					Log: bc.Log,
+				})
+			},
+			OnWait:   DefaultOnWait,
+			WaitTime: DefaultWaitTime,
+		})
+	})
+}
+
+// NewBackgroundOperation creates a new background operation or test that can be
+// notified to stop its operation.
+func NewBackgroundOperation(name string, setup func(c Context),
+	handler func(bc BackgroundContext)) BackgroundOperation {
+	return &simpleBackgroundOperation{
+		name:    name,
+		setup:   setup,
+		handler: handler,
+	}
+}
+
+// WaitForStopEvent will wait until upgrade suite sends a stop event to it.
+// After that happen a handler is invoked to verify environment state and report
+// failures.
+func WaitForStopEvent(bc BackgroundContext, w WaitForStopEventConfiguration) {
+	log := bc.Log
+	for {
+		select {
+		case stopEvent := <-bc.Stop:
+			log.Infof("%s have received a stop event: %s", w.Name, stopEvent.Name())
+			w.OnStop(stopEvent)
+			close(stopEvent.Finished)
+			return
+		default:
+			w.OnWait(bc, w)
+		}
+		time.Sleep(w.WaitTime)
+	}
+}
+
+func (c Configuration) logger() *zap.SugaredLogger {
+	return c.Log.Sugar()
+}
+
+// Name returns a friendly human readable text.
+func (s *StopEvent) Name() string {
+	return s.name
+}
+
+func enrichSuite(s *Suite) *enrichedSuite {
+	es := &enrichedSuite{
+		installations: s.Installations,
+		tests: enrichedTests{
+			preUpgrade:    s.Tests.PreUpgrade,
+			postUpgrade:   s.Tests.PostUpgrade,
+			postDowngrade: s.Tests.PostDowngrade,
+			continual:     make([]stoppableOperation, len(s.Tests.Continual)),
+		},
+	}
+	for i, test := range s.Tests.Continual {
+		es.tests.continual[i] = stoppableOperation{
+			BackgroundOperation: test,
+			stop:                make(chan StopEvent),
+		}
+	}
+	return es
+}
+
+// Name is a human readable operation title, and it will be used in t.Run.
+func (h *simpleOperation) Name() string {
+	return h.name
+}
+
+// Handler is a function that will be called to perform an operation.
+func (h *simpleOperation) Handler() func(c Context) {
+	return h.handler
+}
+
+// Name is a human readable operation title, and it will be used in t.Run.
+func (s *simpleBackgroundOperation) Name() string {
+	return s.name
+}
+
+// Setup method may be used to set up environment before upgrade/downgrade is
+// performed.
+func (s *simpleBackgroundOperation) Setup() func(c Context) {
+	return s.setup
+}
+
+// Handler will be executed in background while upgrade/downgrade is being
+// executed. It can be used to constantly validate environment during that
+// time and/or wait for StopEvent being sent. After StopEvent is received
+// user should validate environment, clean up resources, and report found
+// issues to testing.T forwarded in StepEvent.
+func (s *simpleBackgroundOperation) Handler() func(bc BackgroundContext) {
+	return s.handler
+}