diff --git a/test/upgrade/asserts_test.go b/test/upgrade/asserts_test.go new file mode 100644 index 0000000000..8286568d68 --- /dev/null +++ b/test/upgrade/asserts_test.go @@ -0,0 +1,44 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import ( + "reflect" + "strings" + "testing" +) + +type assertions struct { + t *testing.T +} + +func (a assertions) textContains(haystack string, needles texts) { + for _, needle := range needles.elms { + if !strings.Contains(haystack, needle) { + a.t.Errorf( + "expected \"%s\" is not in: `%s`", + needle, haystack, + ) + } + } +} + +func (a assertions) arraysEqual(actual []string, expected []string) { + if !reflect.DeepEqual(actual, expected) { + a.t.Errorf("arrays differ:\n actual: %#v\nexpected: %#v", actual, expected) + } +} diff --git a/test/upgrade/execute_failures_test.go b/test/upgrade/execute_failures_test.go new file mode 100644 index 0000000000..aae11f1fe2 --- /dev/null +++ b/test/upgrade/execute_failures_test.go @@ -0,0 +1,90 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import ( + "fmt" + "io/ioutil" + "os" + "testing" +) + +func TestSuiteExecuteWithFailures(t *testing.T) { + for i := 1; i <= 8; i++ { + for j := 1; j <= 2; j++ { + fp := failurePoint{ + step: i, + element: j, + } + testSuiteExecuteWithFailingStep(fp, t) + } + } +} + +var allTestsFilter = func(_, _ string) (bool, error) { return true, nil } + +func testSuiteExecuteWithFailingStep(fp failurePoint, t *testing.T) { + assert := assertions{t: t} + testName := fmt.Sprintf("FailAt-%d-%d", fp.step, fp.element) + t.Run(testName, func(t *testing.T) { + var output string + suite := completeSuiteExample(fp) + txt := expectedTexts(suite, fp) + txt.append(upgradeTestRunning, upgradeTestFailure) + log, buf := newExampleZap() + + it := []testing.InternalTest{{ + Name: testName, + F: func(t *testing.T) { + c, _ := newConfig(t) + c.Log = log + suite.Execute(c) + }, + }} + var ok bool + testOutput := captureStdOutput(func() { + ok = testing.RunTests(allTestsFilter, it) + }) + output = buf.String() + + if ok { + t.Fatal("didn't failed, but should") + } + + assert.textContains(output, txt) + assert.textContains(testOutput, texts{ + elms: []string{ + fmt.Sprintf("--- FAIL: FailAt-%d-%d", fp.step, fp.element), + }, + }) + }) +} + +func captureStdOutput(call func()) string { + rescueStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + defer func() { + os.Stdout = rescueStdout + }() + + call() + + _ = w.Close() + out, _ := ioutil.ReadAll(r) + return string(out) +} diff --git a/test/upgrade/functions.go b/test/upgrade/functions.go new file mode 100644 index 0000000000..c6549baf72 --- /dev/null +++ b/test/upgrade/functions.go @@ -0,0 +1,155 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "time" + + "go.uber.org/zap" +) + +// Execute the Suite of upgrade tests with a Configuration given. +func (s *Suite) Execute(c Configuration) { + l := c.logger() + se := suiteExecution{ + suite: enrichSuite(s), + configuration: c, + failed: false, + logger: l, + } + l.Info("🏃 Running upgrade test suite...") + + se.execute() + + if !se.failed { + l.Info("🥳🎉 Success! Upgrade suite completed without errors.") + } else { + l.Error("💣🤬💔️ Upgrade suite have failed!") + } +} + +// NewOperation creates a new upgrade operation or test. +func NewOperation(name string, handler func(c Context)) Operation { + return &simpleOperation{name: name, handler: handler} +} + +// NewBackgroundVerification is convenience function to easily setup a +// background operation that will setup environment and then verify environment +// status after receiving a StopEvent. +func NewBackgroundVerification(name string, setup func(c Context), verify func(c Context)) BackgroundOperation { + return NewBackgroundOperation(name, setup, func(bc BackgroundContext) { + WaitForStopEvent(bc, WaitForStopEventConfiguration{ + Name: name, + OnStop: func(event StopEvent) { + verify(Context{ + T: event.T, + Log: bc.Log, + }) + }, + OnWait: DefaultOnWait, + WaitTime: DefaultWaitTime, + }) + }) +} + +// NewBackgroundOperation creates a new background operation or test that can be +// notified to stop its operation. +func NewBackgroundOperation(name string, setup func(c Context), + handler func(bc BackgroundContext)) BackgroundOperation { + return &simpleBackgroundOperation{ + name: name, + setup: setup, + handler: handler, + } +} + +// WaitForStopEvent will wait until upgrade suite sends a stop event to it. +// After that happen a handler is invoked to verify environment state and report +// failures. +func WaitForStopEvent(bc BackgroundContext, w WaitForStopEventConfiguration) { + log := bc.Log + for { + select { + case stopEvent := <-bc.Stop: + log.Infof("%s have received a stop event: %s", w.Name, stopEvent.Name()) + w.OnStop(stopEvent) + close(stopEvent.Finished) + return + default: + w.OnWait(bc, w) + } + time.Sleep(w.WaitTime) + } +} + +func (c Configuration) logger() *zap.SugaredLogger { + return c.Log.Sugar() +} + +// Name returns a friendly human readable text. +func (s *StopEvent) Name() string { + return s.name +} + +func enrichSuite(s *Suite) *enrichedSuite { + es := &enrichedSuite{ + installations: s.Installations, + tests: enrichedTests{ + preUpgrade: s.Tests.PreUpgrade, + postUpgrade: s.Tests.PostUpgrade, + postDowngrade: s.Tests.PostDowngrade, + continual: make([]stoppableOperation, len(s.Tests.Continual)), + }, + } + for i, test := range s.Tests.Continual { + es.tests.continual[i] = stoppableOperation{ + BackgroundOperation: test, + stop: make(chan StopEvent), + } + } + return es +} + +// Name is a human readable operation title, and it will be used in t.Run. +func (h *simpleOperation) Name() string { + return h.name +} + +// Handler is a function that will be called to perform an operation. +func (h *simpleOperation) Handler() func(c Context) { + return h.handler +} + +// Name is a human readable operation title, and it will be used in t.Run. +func (s *simpleBackgroundOperation) Name() string { + return s.name +} + +// Setup method may be used to set up environment before upgrade/downgrade is +// performed. +func (s *simpleBackgroundOperation) Setup() func(c Context) { + return s.setup +} + +// Handler will be executed in background while upgrade/downgrade is being +// executed. It can be used to constantly validate environment during that +// time and/or wait for StopEvent being sent. After StopEvent is received +// user should validate environment, clean up resources, and report found +// issues to testing.T forwarded in StepEvent. +func (s *simpleBackgroundOperation) Handler() func(bc BackgroundContext) { + return s.handler +} diff --git a/test/upgrade/functions_test.go b/test/upgrade/functions_test.go new file mode 100644 index 0000000000..f47cfd9488 --- /dev/null +++ b/test/upgrade/functions_test.go @@ -0,0 +1,141 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import "testing" + +const ( + upgradeTestRunning = "🏃 Running upgrade test suite..." + upgradeTestSuccess = "🥳🎉 Success! Upgrade suite completed without errors." + upgradeTestFailure = "💣🤬💔️ Upgrade suite have failed!" +) + +func TestExpectedTextsForEmptySuite(t *testing.T) { + assert := assertions{t: t} + fp := notFailing + suite := emptySuiteExample() + txt := expectedTexts(suite, fp) + expected := []string{ + "1) 💿 No base installation registered. Skipping.", + "2) ✅️️ No pre upgrade tests registered. Skipping.", + "3) 🔄 No continual tests registered. Skipping.", + "4) 📀 No upgrade operations registered. Skipping.", + "5) ✅️️ No post upgrade tests registered. Skipping.", + "6) 💿 No downgrade operations registered. Skipping.", + "7) ✅️️ No post downgrade tests registered. Skipping.", + } + assert.arraysEqual(txt.elms, expected) +} + +func TestExpectedTextsForCompleteSuite(t *testing.T) { + assert := assertions{t: t} + fp := notFailing + suite := completeSuiteExample(fp) + txt := expectedTexts(suite, fp) + expected := []string{ + "1) 💿 Installing base installations. 2 are registered.", + `1.1) Installing base install of "Serving latest stable release".`, + `1.2) Installing base install of "Eventing latest stable release".`, + "2) ✅️️ Testing functionality before upgrade is performed. 2 tests are registered.", + `2.1) Testing with "Serving pre upgrade test".`, + `2.2) Testing with "Eventing pre upgrade test".`, + "3) 🔄 Starting continual tests. 2 tests are registered.", + `3.1) Starting continual tests of "Serving continual test".`, + `3.2) Starting continual tests of "Eventing continual test".`, + "4) 📀 Upgrading with 2 registered operations.", + `4.1) Upgrading with "Serving HEAD".`, + `4.2) Upgrading with "Eventing HEAD".`, + "5) ✅️️ Testing functionality after upgrade is performed. 2 tests are registered.", + `5.1) Testing with "Serving post upgrade test".`, + `5.2) Testing with "Eventing post upgrade test".`, + "6) 💿 Downgrading with 2 registered operations.", + `6.1) Downgrading with "Serving latest stable release".`, + `6.2) Downgrading with "Eventing latest stable release".`, + "7) ✅️️ Testing functionality after downgrade is performed. 2 tests are registered.", + `7.1) Testing with "Serving post downgrade test".`, + `7.2) Testing with "Eventing post downgrade test".`, + "8) ✋ Verifying 2 running continual tests.", + `8.1) Verifying "Serving continual test".`, + `8.2) Verifying "Eventing continual test".`, + } + assert.arraysEqual(txt.elms, expected) +} + +func TestExpectedTextsForFailingCompleteSuite(t *testing.T) { + assert := assertions{t: t} + fp := failurePoint{ + step: 2, + element: 1, + } + suite := completeSuiteExample(fp) + txt := expectedTexts(suite, fp) + expected := []string{ + "1) 💿 Installing base installations. 2 are registered.", + `1.1) Installing base install of "Serving latest stable release".`, + `1.2) Installing base install of "Eventing latest stable release".`, + "2) ✅️️ Testing functionality before upgrade is performed. 2 tests are registered.", + `2.1) Testing with "FailingOfServing pre upgrade test".`, + } + assert.arraysEqual(txt.elms, expected) +} + +func TestSuiteExecuteEmpty(t *testing.T) { + assert := assertions{t: t} + c, buf := newConfig(t) + fp := notFailing + suite := emptySuiteExample() + suite.Execute(c) + output := buf.String() + if c.T.Failed() { + return + } + + txt := expectedTexts(suite, fp) + txt.append(upgradeTestRunning, upgradeTestSuccess) + + assert.textContains(output, txt) +} + +func TestSuiteExecuteWithComplete(t *testing.T) { + assert := assertions{t: t} + c, buf := newConfig(t) + fp := notFailing + suite := completeSuiteExample(fp) + suite.Execute(c) + output := buf.String() + if c.T.Failed() { + return + } + txt := expectedTexts(suite, fp) + txt.append(upgradeTestRunning, upgradeTestSuccess) + txt.append( + "Installing Serving stable 0.17.1", + "Installing Eventing stable 0.17.2", + "Running Serving continual test", + "Stopping and verify of Eventing continual test", + "Installing Serving HEAD at e3c4563", + "Installing Eventing HEAD at 12f67cc", + "Installing Serving stable 0.17.1", + "Installing Eventing stable 0.17.2", + "Serving have received a stop event", + "Eventing continual test have received a stop event", + "Serving - probing functionality...", + "Eventing continual test - probing functionality...", + ) + + assert.textContains(output, txt) +} diff --git a/test/upgrade/private_types.go b/test/upgrade/private_types.go new file mode 100644 index 0000000000..e34f090050 --- /dev/null +++ b/test/upgrade/private_types.go @@ -0,0 +1,63 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import "go.uber.org/zap" + +type suiteExecution struct { + suite *enrichedSuite + configuration Configuration + failed bool + logger *zap.SugaredLogger +} + +type enrichedSuite struct { + installations Installations + tests enrichedTests +} + +type enrichedTests struct { + preUpgrade []Operation + postUpgrade []Operation + postDowngrade []Operation + continual []stoppableOperation +} + +type stoppableOperation struct { + BackgroundOperation + stop chan StopEvent +} + +type operationGroup struct { + num int + operations []Operation + groupName string + groupTemplate string + elementTemplate string + skippingGroupTemplate string +} + +type simpleOperation struct { + name string + handler func(c Context) +} + +type simpleBackgroundOperation struct { + name string + setup func(c Context) + handler func(bc BackgroundContext) +} diff --git a/test/upgrade/steps.go b/test/upgrade/steps.go new file mode 100644 index 0000000000..c50bdcfb62 --- /dev/null +++ b/test/upgrade/steps.go @@ -0,0 +1,155 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" +) + +const skippingOperationTemplate = `Skipping "%s" as previous operation have failed` + +func (se *suiteExecution) installingBase(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.Base, + groupName: "InstallingBase", + elementTemplate: `%d.%d) Installing base install of "%s".`, + skippingGroupTemplate: "%d) 💿 No base installation registered. Skipping.", + groupTemplate: "%d) 💿 Installing base installations. %d are registered.", + }) +} + +func (se *suiteExecution) preUpgradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.preUpgrade, + groupName: "PreUpgradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No pre upgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality before upgrade is performed." + + " %d tests are registered.", + }) +} + +func (se *suiteExecution) startContinualTests(num int) { + l := se.logger + operations := se.suite.tests.continual + groupTemplate := "%d) 🔄 Starting continual tests. " + + "%d tests are registered." + elementTemplate := `%d.%d) Starting continual tests of "%s".` + numOps := len(operations) + se.configuration.T.Run("ContinualTests", func(t *testing.T) { + if numOps > 0 { + l.Infof(groupTemplate, num, numOps) + for i := range operations { + operation := operations[i] + l.Infof(elementTemplate, num, i+1, operation.Name()) + if se.failed { + l.Debugf(skippingOperationTemplate, operation.Name()) + return + } + setup := operation.Setup() + t.Run("Setup"+operation.Name(), func(t *testing.T) { + setup(Context{T: t, Log: l}) + }) + handler := operation.Handler() + go func() { + bc := BackgroundContext{Log: l, Stop: operation.stop} + handler(bc) + }() + + se.failed = se.failed || t.Failed() + if se.failed { + return + } + } + + } else { + l.Infof("%d) 🔄 No continual tests registered. Skipping.", num) + } + }) +} + +func (se *suiteExecution) verifyContinualTests(num int) { + l := se.logger + testsCount := len(se.suite.tests.continual) + if testsCount > 0 { + se.configuration.T.Run("VerifyContinualTests", func(t *testing.T) { + l.Infof("%d) ✋ Verifying %d running continual tests.", num, testsCount) + for i, operation := range se.suite.tests.continual { + t.Run(operation.Name(), func(t *testing.T) { + l.Infof(`%d.%d) Verifying "%s".`, num, i+1, operation.Name()) + finished := make(chan struct{}) + operation.stop <- StopEvent{ + T: t, + Finished: finished, + name: "Stop of " + operation.Name(), + } + <-finished + se.failed = se.failed || t.Failed() + l.Debugf(`Finished "%s"`, operation.Name()) + }) + } + }) + } +} + +func (se *suiteExecution) upgradeWith(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.UpgradeWith, + groupName: "UpgradeWith", + elementTemplate: `%d.%d) Upgrading with "%s".`, + skippingGroupTemplate: "%d) 📀 No upgrade operations registered. Skipping.", + groupTemplate: "%d) 📀 Upgrading with %d registered operations.", + }) +} + +func (se *suiteExecution) postUpgradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.postUpgrade, + groupName: "PostUpgradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No post upgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality after upgrade is performed." + + " %d tests are registered.", + }) +} + +func (se *suiteExecution) downgradeWith(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.DowngradeWith, + groupName: "DowngradeWith", + elementTemplate: `%d.%d) Downgrading with "%s".`, + skippingGroupTemplate: "%d) 💿 No downgrade operations registered. Skipping.", + groupTemplate: "%d) 💿 Downgrading with %d registered operations.", + }) +} + +func (se *suiteExecution) postDowngradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.postDowngrade, + groupName: "PostDowngradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No post downgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality after downgrade is performed." + + " %d tests are registered.", + }) +} diff --git a/test/upgrade/suite_examples_test.go b/test/upgrade/suite_examples_test.go new file mode 100644 index 0000000000..390b9e313e --- /dev/null +++ b/test/upgrade/suite_examples_test.go @@ -0,0 +1,162 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import ( + "time" + + "knative.dev/pkg/test/upgrade" +) + +const ( + shortWait = 50 * time.Microsecond + longWait = 750 * time.Microsecond +) + +func init() { + upgrade.DefaultOnWait = func(bc upgrade.BackgroundContext, self upgrade.WaitForStopEventConfiguration) { + bc.Log.Debugf("%s - probing functionality...", self.Name) + } + upgrade.DefaultWaitTime = shortWait +} + +var ( + notFailing = failurePoint{step: -1, element: -1} + messageFormatters = messageFormatterRepository{ + baseInstall: createMessages(formats{ + starting: "%d) 💿 Installing base installations. %d are registered.", + element: `%d.%d) Installing base install of "%s".`, + skipped: "%d) 💿 No base installation registered. Skipping.", + }), + preUpgrade: createMessages(formats{ + starting: "%d) ✅️️ Testing functionality before upgrade is performed. %d tests are registered.", + element: `%d.%d) Testing with "%s".`, + skipped: "%d) ✅️️ No pre upgrade tests registered. Skipping.", + }), + startContinual: createMessages(formats{ + starting: "%d) 🔄 Starting continual tests. %d tests are registered.", + element: `%d.%d) Starting continual tests of "%s".`, + skipped: "%d) 🔄 No continual tests registered. Skipping.", + }), + upgrade: createMessages(formats{ + starting: "%d) 📀 Upgrading with %d registered operations.", + element: `%d.%d) Upgrading with "%s".`, + skipped: "%d) 📀 No upgrade operations registered. Skipping.", + }), + postUpgrade: createMessages(formats{ + starting: "%d) ✅️️ Testing functionality after upgrade is performed. %d tests are registered.", + element: `%d.%d) Testing with "%s".`, + skipped: "%d) ✅️️ No post upgrade tests registered. Skipping.", + }), + downgrade: createMessages(formats{ + starting: "%d) 💿 Downgrading with %d registered operations.", + element: `%d.%d) Downgrading with "%s".`, + skipped: "%d) 💿 No downgrade operations registered. Skipping.", + }), + postDowngrade: createMessages(formats{ + starting: "%d) ✅️️ Testing functionality after downgrade is performed. %d tests are registered.", + element: `%d.%d) Testing with "%s".`, + skipped: "%d) ✅️️ No post downgrade tests registered. Skipping.", + }), + verifyContinual: createMessages(formats{ + starting: "%d) ✋ Verifying %d running continual tests.", + element: `%d.%d) Verifying "%s".`, + skipped: "", + }), + } + serving = component{ + installs: installs{ + stable: upgrade.NewOperation("Serving latest stable release", func(c upgrade.Context) { + c.Log.Info("Installing Serving stable 0.17.1") + time.Sleep(longWait) + }), + head: upgrade.NewOperation("Serving HEAD", func(c upgrade.Context) { + c.Log.Info("Installing Serving HEAD at e3c4563") + time.Sleep(longWait) + }), + }, + tests: tests{ + preUpgrade: upgrade.NewOperation("Serving pre upgrade test", func(c upgrade.Context) { + c.Log.Info("Running Serving pre upgrade test") + time.Sleep(shortWait) + }), + postUpgrade: upgrade.NewOperation("Serving post upgrade test", func(c upgrade.Context) { + c.Log.Info("Running Serving post upgrade test") + time.Sleep(shortWait) + }), + postDowngrade: upgrade.NewOperation("Serving post downgrade test", func(c upgrade.Context) { + c.Log.Info("Running Serving post downgrade test") + time.Sleep(shortWait) + }), + continual: upgrade.NewBackgroundOperation("Serving continual test", + func(c upgrade.Context) { + c.Log.Info("Setup of Serving continual test") + time.Sleep(shortWait) + }, + func(bc upgrade.BackgroundContext) { + bc.Log.Info("Running Serving continual test") + upgrade.WaitForStopEvent(bc, upgrade.WaitForStopEventConfiguration{ + Name: "Serving", + OnStop: func(event upgrade.StopEvent) { + bc.Log.Info("Stopping and verify of Serving continual test") + time.Sleep(shortWait) + }, + OnWait: func(bc upgrade.BackgroundContext, self upgrade.WaitForStopEventConfiguration) { + bc.Log.Debugf("%s - probing functionality...", self.Name) + }, + WaitTime: shortWait, + }) + }), + }, + } + eventing = component{ + installs: installs{ + stable: upgrade.NewOperation("Eventing latest stable release", func(c upgrade.Context) { + c.Log.Info("Installing Eventing stable 0.17.2") + time.Sleep(longWait) + }), + head: upgrade.NewOperation("Eventing HEAD", func(c upgrade.Context) { + c.Log.Info("Installing Eventing HEAD at 12f67cc") + time.Sleep(longWait) + }), + }, + tests: tests{ + preUpgrade: upgrade.NewOperation("Eventing pre upgrade test", func(c upgrade.Context) { + c.Log.Info("Running Eventing pre upgrade test") + time.Sleep(shortWait) + }), + postUpgrade: upgrade.NewOperation("Eventing post upgrade test", func(c upgrade.Context) { + c.Log.Info("Running Eventing post upgrade test") + time.Sleep(shortWait) + }), + postDowngrade: upgrade.NewOperation("Eventing post downgrade test", func(c upgrade.Context) { + c.Log.Info("Running Eventing post downgrade test") + time.Sleep(shortWait) + }), + continual: upgrade.NewBackgroundVerification("Eventing continual test", + func(c upgrade.Context) { + c.Log.Info("Setup of Eventing continual test") + time.Sleep(shortWait) + }, + func(c upgrade.Context) { + c.Log.Info("Stopping and verify of Eventing continual test") + time.Sleep(shortWait) + }, + ), + }, + } +) diff --git a/test/upgrade/suite_execution.go b/test/upgrade/suite_execution.go new file mode 100644 index 0000000000..70c5ea095e --- /dev/null +++ b/test/upgrade/suite_execution.go @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" +) + +func (se *suiteExecution) processOperationGroup(op operationGroup) { + l := se.logger + se.configuration.T.Run(op.groupName, func(t *testing.T) { + if len(op.operations) > 0 { + l.Infof(op.groupTemplate, op.num, len(op.operations)) + for i, operation := range op.operations { + l.Infof(op.elementTemplate, op.num, i+1, operation.Name()) + if se.failed { + l.Debugf(skippingOperationTemplate, operation.Name()) + return + } + handler := operation.Handler() + t.Run(operation.Name(), func(t *testing.T) { + handler(Context{T: t, Log: l}) + }) + se.failed = se.failed || t.Failed() + if se.failed { + return + } + } + } else { + l.Infof(op.skippingGroupTemplate, op.num) + } + }) +} + +func (se *suiteExecution) execute() { + idx := 1 + operations := []func(num int){ + se.installingBase, + se.preUpgradeTests, + } + for _, operation := range operations { + operation(idx) + idx++ + if se.failed { + return + } + } + + se.startContinualTests(idx) + idx++ + if se.failed { + return + } + defer func() { + se.verifyContinualTests(idx) + }() + + operations = []func(num int){ + se.upgradeWith, + se.postUpgradeTests, + se.downgradeWith, + se.postDowngradeTests, + } + for _, operation := range operations { + operation(idx) + idx++ + if se.failed { + return + } + } +} diff --git a/test/upgrade/testing_operations_test.go b/test/upgrade/testing_operations_test.go new file mode 100644 index 0000000000..79483d7a49 --- /dev/null +++ b/test/upgrade/testing_operations_test.go @@ -0,0 +1,293 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import ( + "bytes" + "fmt" + "os" + "sync" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest" + "knative.dev/pkg/test/upgrade" +) + +func newConfig(t *testing.T) (upgrade.Configuration, fmt.Stringer) { + log, buf := newExampleZap() + c := upgrade.Configuration{T: t, Log: log} + return c, buf +} + +func newExampleZap() (*zap.Logger, fmt.Stringer) { + ec := zap.NewDevelopmentEncoderConfig() + ec.TimeKey = "" + encoder := zapcore.NewConsoleEncoder(ec) + buf := &buffer{ + Buffer: bytes.Buffer{}, + Mutex: sync.Mutex{}, + Syncer: zaptest.Syncer{}, + } + ws := zapcore.NewMultiWriteSyncer(buf, os.Stdout) + core := zapcore.NewCore(encoder, ws, zap.DebugLevel) + return zap.New(core).WithOptions(), buf +} + +func createSteps(s upgrade.Suite) []*step { + continualTestsGeneralized := generalizeOpsFromBg(s.Tests.Continual) + return []*step{{ + messages: messageFormatters.baseInstall, + ops: generalizeOps(s.Installations.Base), + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Installations.Base = ops.asOperations() + }, + }, { + messages: messageFormatters.preUpgrade, + ops: generalizeOps(s.Tests.PreUpgrade), + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Tests.PreUpgrade = ops.asOperations() + }, + }, { + messages: messageFormatters.startContinual, + ops: continualTestsGeneralized, + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Tests.Continual = ops.asBackgroundOperation() + }, + }, { + messages: messageFormatters.upgrade, + ops: generalizeOps(s.Installations.UpgradeWith), + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Installations.UpgradeWith = ops.asOperations() + }, + }, { + messages: messageFormatters.postUpgrade, + ops: generalizeOps(s.Tests.PostUpgrade), + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Tests.PostUpgrade = ops.asOperations() + }, + }, { + messages: messageFormatters.downgrade, + ops: generalizeOps(s.Installations.DowngradeWith), + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Installations.DowngradeWith = ops.asOperations() + }, + }, { + messages: messageFormatters.postDowngrade, + ops: generalizeOps(s.Tests.PostDowngrade), + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Tests.PostDowngrade = ops.asOperations() + }, + }, { + messages: messageFormatters.verifyContinual, + ops: continualTestsGeneralized, + updateSuite: func(ops operations, s *upgrade.Suite) { + s.Tests.Continual = ops.asBackgroundOperation() + }, + }} +} + +func expectedTexts(s upgrade.Suite, fp failurePoint) texts { + steps := createSteps(s) + tt := texts{elms: nil} + for i, st := range steps { + stepIdx := i + 1 + if st.ops.length() == 0 { + tt.append(st.skipped(stepIdx)) + } else { + tt.append(st.starting(stepIdx, st.ops.length())) + for j, op := range st.ops.ops { + elemIdx := j + 1 + tt.append(st.element(stepIdx, elemIdx, op.Name())) + if fp.step == stepIdx && fp.element == elemIdx { + return tt + } + } + } + } + return tt +} + +func generalizeOps(ops []upgrade.Operation) operations { + gen := make([]*operation, len(ops)) + for idx, op := range ops { + gen[idx] = &operation{op: op} + } + return operations{ops: gen} +} + +func generalizeOpsFromBg(ops []upgrade.BackgroundOperation) operations { + gen := make([]*operation, len(ops)) + for idx, op := range ops { + gen[idx] = &operation{bg: op} + } + return operations{ops: gen} +} + +func createMessages(mf formats) messages { + return messages{ + skipped: func(args ...interface{}) string { + empty := "" + if mf.skipped == empty { + return empty + } + return fmt.Sprintf(mf.skipped, args...) + }, + starting: func(args ...interface{}) string { + return fmt.Sprintf(mf.starting, args...) + }, + element: func(args ...interface{}) string { + return fmt.Sprintf(mf.element, args...) + }, + } +} + +func (tt *texts) append(messages ...string) { + for _, msg := range messages { + if msg == "" { + continue + } + tt.elms = append(tt.elms, msg) + } +} + +func completeSuiteExample(fp failurePoint) upgrade.Suite { + suite := upgrade.Suite{ + Tests: upgrade.Tests{ + PreUpgrade: []upgrade.Operation{ + serving.tests.preUpgrade, eventing.tests.preUpgrade, + }, + PostUpgrade: []upgrade.Operation{ + serving.tests.postUpgrade, eventing.tests.postUpgrade, + }, + PostDowngrade: []upgrade.Operation{ + serving.tests.postDowngrade, eventing.tests.postDowngrade, + }, + Continual: []upgrade.BackgroundOperation{ + serving.tests.continual, eventing.tests.continual, + }, + }, + Installations: upgrade.Installations{ + Base: []upgrade.Operation{ + serving.installs.stable, eventing.installs.stable, + }, + UpgradeWith: []upgrade.Operation{ + serving.installs.head, eventing.installs.head, + }, + DowngradeWith: []upgrade.Operation{ + serving.installs.stable, eventing.installs.stable, + }, + }, + } + return enrichSuiteWithFailures(suite, fp) +} + +func emptySuiteExample() upgrade.Suite { + return upgrade.Suite{ + Tests: upgrade.Tests{}, + Installations: upgrade.Installations{}, + } +} + +func enrichSuiteWithFailures(suite upgrade.Suite, fp failurePoint) upgrade.Suite { + steps := createSteps(suite) + for i, st := range steps { + for j, op := range st.ops.ops { + if fp.step == i+1 && fp.element == j+1 { + op.fail(fp.step == 3) + } + } + } + return recreateSuite(steps) +} + +func recreateSuite(steps []*step) upgrade.Suite { + suite := &upgrade.Suite{ + Tests: upgrade.Tests{}, + Installations: upgrade.Installations{}, + } + for _, st := range steps { + st.updateSuite(st.ops, suite) + } + return *suite +} + +func (o operation) Name() string { + if o.op != nil { + return o.op.Name() + } + return o.bg.Name() +} + +func (o *operation) fail(setupFail bool) { + failureTestingMessage := "This error is expected to be seen. Upgrade suite should fail." + testName := fmt.Sprintf("FailingOf%s", o.Name()) + if o.op != nil { + prev := o.op + o.op = upgrade.NewOperation(testName, func(c upgrade.Context) { + handler := prev.Handler() + handler(c) + c.T.Error(failureTestingMessage) + c.Log.Error(failureTestingMessage) + }) + } else { + prev := o.bg + o.bg = upgrade.NewBackgroundOperation(testName, func(c upgrade.Context) { + handler := prev.Setup() + handler(c) + if setupFail { + c.T.Error(failureTestingMessage) + c.Log.Error(failureTestingMessage) + } + }, func(bc upgrade.BackgroundContext) { + upgrade.WaitForStopEvent(bc, upgrade.WaitForStopEventConfiguration{ + Name: testName, + OnStop: func(event upgrade.StopEvent) { + if !setupFail { + event.T.Error(failureTestingMessage) + bc.Log.Error(failureTestingMessage) + } + }, + OnWait: func(bc upgrade.BackgroundContext, self upgrade.WaitForStopEventConfiguration) { + bc.Log.Debugf("%s - probing functionality...", self.Name) + }, + WaitTime: shortWait, + }) + }) + } +} + +func (o operations) length() int { + return len(o.ops) +} + +func (o operations) asOperations() []upgrade.Operation { + ops := make([]upgrade.Operation, o.length()) + for i, op := range o.ops { + ops[i] = op.op + } + return ops +} + +func (o operations) asBackgroundOperation() []upgrade.BackgroundOperation { + ops := make([]upgrade.BackgroundOperation, o.length()) + for i, op := range o.ops { + ops[i] = op.bg + } + return ops +} diff --git a/test/upgrade/testing_types_test.go b/test/upgrade/testing_types_test.go new file mode 100644 index 0000000000..0dabca141e --- /dev/null +++ b/test/upgrade/testing_types_test.go @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import "knative.dev/pkg/test/upgrade" + +type failurePoint struct { + step int + element int +} + +type texts struct { + elms []string +} + +type messageFormatter func(args ...interface{}) string + +type step struct { + messages + ops operations + updateSuite func(ops operations, s *upgrade.Suite) +} + +type operations struct { + ops []*operation +} + +type operation struct { + op upgrade.Operation + bg upgrade.BackgroundOperation +} + +type formats struct { + skipped string + starting string + element string +} + +type messages struct { + starting messageFormatter + element messageFormatter + skipped messageFormatter +} + +type messageFormatterRepository struct { + baseInstall messages + preUpgrade messages + startContinual messages + upgrade messages + postUpgrade messages + downgrade messages + postDowngrade messages + verifyContinual messages +} + +type component struct { + installs + tests +} + +type installs struct { + stable upgrade.Operation + head upgrade.Operation +} + +type tests struct { + preUpgrade upgrade.Operation + postUpgrade upgrade.Operation + continual upgrade.BackgroundOperation + postDowngrade upgrade.Operation +} diff --git a/test/upgrade/types.go b/test/upgrade/types.go new file mode 100644 index 0000000000..afcccec332 --- /dev/null +++ b/test/upgrade/types.go @@ -0,0 +1,123 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" + "time" + + "go.uber.org/zap" +) + +// Suite represents a upgrade tests suite that can be executed and will perform +// execution in predictable manner. +type Suite struct { + Tests Tests + Installations Installations +} + +// Tests holds a list of operations for various part of upgrade suite. +type Tests struct { + PreUpgrade []Operation + PostUpgrade []Operation + PostDowngrade []Operation + Continual []BackgroundOperation +} + +// Installations holds a list of operations that will install Knative components +// in different versions. +type Installations struct { + Base []Operation + UpgradeWith []Operation + DowngradeWith []Operation +} + +// Operation represents a upgrade test operation like test or installation that +// can be provided by specific component or reused in aggregating components. +type Operation interface { + // Name is a human readable operation title, and it will be used in t.Run. + Name() string + // Handler is a function that will be called to perform an operation. + Handler() func(c Context) +} + +// BackgroundOperation represents a upgrade test operation that will be +// performed in background while other operations is running. To achieve that +// a passed BackgroundContext should be used to synchronize it's operations with +// Ready and Stop channels. +type BackgroundOperation interface { + // Name is a human readable operation title, and it will be used in t.Run. + Name() string + // Setup method may be used to set up environment before upgrade/downgrade is + // performed. + Setup() func(c Context) + // Handler will be executed in background while upgrade/downgrade is being + // executed. It can be used to constantly validate environment during that + // time and/or wait for StopEvent being sent. After StopEvent is received + // user should validate environment, clean up resources, and report found + // issues to testing.T forwarded in StepEvent. + Handler() func(bc BackgroundContext) +} + +// Context is an object that is passed to every operation. It contains testing.T +// for error reporting and zap.SugaredLogger for unbuffered logging. +type Context struct { + T *testing.T + Log *zap.SugaredLogger +} + +// BackgroundContext is a upgrade test execution context that will be passed +// down to each handler of BackgroundOperation. It contains a StopEvent channel +// which end user should use to obtain a testing.T for error reporting. Until +// StopEvent is sent user may use zap.SugaredLogger to log state of execution if +// necessary. +type BackgroundContext struct { + Log *zap.SugaredLogger + Stop <-chan StopEvent +} + +// StopEvent represents an event that is to be received by background operation +// to indicate that is should stop it's operations and validate results using +// passed T. User should use Finished channel to signalize upgrade suite that +// all stop & verify operations are finished and it is safe to end tests. +type StopEvent struct { + T *testing.T + Finished chan<- struct{} + name string +} + +// WaitForStopEventConfiguration holds a values to be used be WaitForStopEvent +// function. OnStop will be called when StopEvent is sent. OnWait will be +// invoked in a loop while waiting, and each wait act is driven by WaitTime +// amount. +type WaitForStopEventConfiguration struct { + Name string + OnStop func(event StopEvent) + OnWait func(bc BackgroundContext, self WaitForStopEventConfiguration) + WaitTime time.Duration +} + +// Configuration holds required and optional configuration to run upgrade tests. +type Configuration struct { + T *testing.T + Log *zap.Logger +} + +// SuiteExecutor is to execute upgrade test suite. +type SuiteExecutor interface { + Execute(c Configuration) +} diff --git a/test/upgrade/vars.go b/test/upgrade/vars.go new file mode 100644 index 0000000000..8a8ba93e11 --- /dev/null +++ b/test/upgrade/vars.go @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import "time" + +var ( + // DefaultWaitTime holds a default value for WaitForStopEventConfiguration + // when used within a NewBackgroundVerification function. + DefaultWaitTime = 20 * time.Millisecond + + // DefaultOnWait is a implementation that will be called by default for each + // wait performed by WaitForStopEvent when used within + // NewBackgroundVerification function. + DefaultOnWait = func(bc BackgroundContext, self WaitForStopEventConfiguration) { + // do nothing by default + } +) diff --git a/test/upgrade/zaptest_buffer_test.go b/test/upgrade/zaptest_buffer_test.go new file mode 100644 index 0000000000..05b373743a --- /dev/null +++ b/test/upgrade/zaptest_buffer_test.go @@ -0,0 +1,49 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade_test + +import ( + "bytes" + "sync" + + "go.uber.org/zap/zaptest" +) + +// To avoid race condition on zaptest.Buffer, see: https://stackoverflow.com/a/36226525/844449 +type buffer struct { + bytes.Buffer + sync.Mutex + zaptest.Syncer +} + +func (b *buffer) Read(p []byte) (n int, err error) { + b.Mutex.Lock() + defer b.Mutex.Unlock() + return b.Buffer.Read(p) +} + +func (b *buffer) Write(p []byte) (n int, err error) { + b.Mutex.Lock() + defer b.Mutex.Unlock() + return b.Buffer.Write(p) +} + +func (b *buffer) String() string { + b.Mutex.Lock() + defer b.Mutex.Unlock() + return b.Buffer.String() +}