Feature: reuse e2e tests on productized code

.gitignore

@@ -1,6 +1,9 @@
 # Temporary Build Files
 build/_output
 build/_test
+# Goland
+.idea
+
 # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode
 ### Emacs ###
 # -*- mode: gitignore; -*-

cmd/manager/kodata/knative-serving/dummy.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package knative_serving is a dummy buildable source file to satisfy `dep`
+package knative_serving

hack/update-deps.sh

@@ -20,13 +20,13 @@ set -o pipefail
 
 source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/library.sh
 
-cd ${REPO_ROOT_DIR}
+cd "${REPO_ROOT_DIR}"
 
 # Ensure we have everything we need under vendor/
 dep ensure
 
-rm -rf $(find vendor/ -name 'OWNERS')
-rm -rf $(find vendor/ -name '*_test.go')
+find vendor/ -name 'OWNERS' -delete
+find vendor/ -name '*_test.go' -delete
 
 update_licenses third_party/VENDOR-LICENSE "./cmd/*"
 

test/context.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2019 The Knative Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import "testing"
+
+// Context represents a testing context that can be tuned to override specific
+// parts of a suite
+type Context struct {
+	t           *testing.T
+	overrides   []Specification
+	activeTests []string
+}
+
+// T returns standard testing.T
+func (ctx Context) T() *testing.T {
+	return ctx.t
+}
+
+// Runner creates a new runner based on a given context
+func (ctx Context) Runner() Runner {
+	return contextRunner{
+		ctx: &ctx,
+	}
+}
+
+// RunSuite executes given test or execute it's override
+func (runner contextRunner) Run(name string, testfunc func(t *testing.T)) bool {
+	ctx := runner.ctx
+	t := ctx.t
+	for _, spec := range ctx.overrides {
+		if spec.matchesNameInContext(name, ctx) {
+			t.Logf("Overriding %s test", spec.name())
+			return t.Run(name, spec.testfunc(ctx))
+		}
+	}
+	return t.Run(name, testfunc)
+}
+
+// WithOverride adds specification to be executed instead of given test
+func (ctx *Context) WithOverride(spec Specification) *Context {
+	ctx.overrides = append(ctx.overrides, spec)
+	return ctx
+}
+
+// RunSuite will run a test suite within given context
+func (ctx *Context) RunSuite(suite []Specification) {
+	for _, spec := range suite {
+		spec.run(ctx)
+	}
+}
+
+// NewContext creates a new context
+func NewContext(t *testing.T) *Context {
+	return &Context{
+		t:           t,
+		overrides:   make([]Specification, 0),
+		activeTests: make([]string, 0),
+	}
+}
+
+func (ctx *Context) push(testname string) {
+	ctx.activeTests = append(ctx.activeTests, testname)
+}
+
+func (ctx *Context) pop() string {
+	// Top element
+	n := len(ctx.activeTests) - 1
+	testname := ctx.activeTests[n]
+	ctx.activeTests = ctx.activeTests[:n]
+	return testname
+}
+
+type contextRunner struct {
+	ctx *Context
+}

test/e2e/compliance.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2019 The Knative Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import "knative.dev/serving-operator/test"
+
+// ComplianceSuite can be executed by productized code's test plan to assert
+// compliance with upstream knative
+func ComplianceSuite() []test.Specification {
+	return suite
+}
+
+var suite = []test.Specification{
+	test.NewContextualSpec("TestKnativeServingDeployment", testKnativeServingDeployment),
+}

test/e2e/compliance_test.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2019 The Knative Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"knative.dev/serving-operator/test"
+	"testing"
+)
+
+func TestComplianceSuite(t *testing.T) {
+	suite := ComplianceSuite()
+
+	if len(suite) <= 0 {
+		t.Error("There should be some tests that are exported as compliance suite")
+	}
+}
+
+var runState map[string]int
+
+const exampleJira = "XXXX-1234"
+
+func TestSkipOfSpecificPartOfTestSuite(t *testing.T) {
+	suite := exampleComplianceSuite()
+
+	runState = map[string]int{
+		"alpha": 0,
+		"beta":  0,
+		"gamma": 0,
+	}
+
+	test.
+		NewContext(t).
+		WithOverride(test.Skipf("TestParent/beta", "Skip due to %v", exampleJira)).
+		RunSuite(suite)
+
+	if runState["alpha"] != 1 {
+		t.Error("Alpha should be executed just once")
+	}
+	if runState["beta"] != 0 {
+		t.Error("Beta should be skipped, but wasn't")
+	}
+	if runState["gamma"] != 1 {
+		t.Error("Gamma should be executed just once")
+	}
+}
+
+func exampleComplianceSuite() []test.Specification {
+	return []test.Specification{
+		test.NewContextualSpec("TestParent", testParent),
+	}
+}
+
+func testParent(ctx *test.Context) {
+	r := ctx.Runner()
+
+	r.Run("alpha", func(t *testing.T) {
+		t.Log("Alpha is OK")
+		runState["alpha"]++
+	})
+
+	r.Run("beta", func(t *testing.T) {
+		runState["beta"]++
+		t.Errorf("Beta is failing, because %v", exampleJira)
+	})
+
+	r.Run("gamma", func(t *testing.T) {
+		t.Log("Gamma is OK")
+		runState["gamma"]++
+	})
+}