From 79708bd739a2d7f0af28b6335e3437876f0d5df8 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Mon, 3 May 2021 14:43:21 +0200 Subject: [PATCH 01/10] Use test namespaces from pre-defined range --- test/lib/test_runner.go | 136 ++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 41 deletions(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index c4cc2faa020..df43951cc45 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -18,10 +18,13 @@ package lib import ( "context" + "errors" "fmt" + "os" "path/filepath" "sort" "strings" + "sync" "testing" "time" @@ -29,10 +32,7 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/storage/names" - pkgTest "knative.dev/pkg/test" - "knative.dev/pkg/test/helpers" "knative.dev/pkg/test/prow" "knative.dev/eventing/pkg/utils" @@ -47,6 +47,15 @@ import ( const ( podLogsDir = "pod-logs" testPullSecretName = "kn-eventing-test-pull-secret" + MaxNamespaces = 20 + MaxRetries = 10 + RetrySleepDuration = 5 * time.Second +) + +var ( + nsMutex sync.Mutex + serviceCount int + namespaceCount int ) // ComponentsTestRunner is used to run tests against different eventing components. @@ -148,19 +157,12 @@ var SetupClientOptionNoop SetupClientOption = func(*Client) { // Setup creates the client objects needed in the e2e tests, // and does other setups, like creating namespaces, set the test case to run in parallel, etc. func Setup(t *testing.T, runInParallel bool, options ...SetupClientOption) *Client { - // Create a new namespace to run this test case. - namespace := makeK8sNamespace(t.Name()) - t.Logf("namespace is : %q", namespace) - client, err := NewClient( - pkgTest.Flags.Kubeconfig, - pkgTest.Flags.Cluster, - namespace, - t) + client, err := CreateNamespacedClient(t) if err != nil { t.Fatal("Couldn't initialize clients:", err) } - CreateNamespaceIfNeeded(t, client, namespace) + SetupServiceAccountInClientNamespace(t, client) // Run the test case in parallel if needed. if runInParallel { @@ -178,9 +180,72 @@ func Setup(t *testing.T, runInParallel bool, options ...SetupClientOption) *Clie return client } -func makeK8sNamespace(baseFuncName string) string { - base := helpers.MakeK8sNamePrefix(baseFuncName) - return names.SimpleNameGenerator.GenerateName(base + "-") +func CreateNamespacedClient(t *testing.T) (*Client, error) { + ns := "" + // Try namespaces from pre-defined range before giving up. + for i := 0; i < MaxNamespaces; i++ { + ns = NextNamespace() + client, err := NewClient( + pkgTest.Flags.Kubeconfig, + pkgTest.Flags.Cluster, + ns, + t) + if err != nil { + return nil, err + } + if err := CreateNamespace(client, ns); err != nil { + if apierrs.IsAlreadyExists(err) { + continue + } + return nil, err + } + return client, nil + } + return nil, errors.New("unable to find available namespace in predefined range") +} + +// NextNamespace returns the next unique namespace. +func NextNamespace() string { + ns := os.Getenv("EVENTING_E2E_NAMESPACE") + if ns == "" { + ns = "eventing-e2e" + } + return fmt.Sprintf("%s%d", ns, GetNextNamespaceId()) +} + +// GetNextNamespaceId return the next unique ID for the next namespace. +func GetNextNamespaceId() int { + nsMutex.Lock() + defer nsMutex.Unlock() + current := namespaceCount + namespaceCount++ + return current +} + +// CreateNamespace creates the given namespace. +func CreateNamespace(client *Client, namespace string) error { + err := createNamespaceWithRetry(client, namespace) + if err != nil { + return fmt.Errorf("could not create namespace %s: %w", namespace, err) + } + return nil +} + +func createNamespaceWithRetry(client *Client, namespace string) error { + var ( + retries int + err error + ) + for retries < MaxRetries { + nsSpec := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + if _, err = client.Kube.CoreV1().Namespaces(). + Create(context.Background(), nsSpec, metav1.CreateOptions{}); err == nil { + return nil + } + retries++ + time.Sleep(RetrySleepDuration) + } + return err } // TearDown will delete created names using clients. @@ -253,33 +318,22 @@ func formatEvent(e *corev1.Event) string { }, "\n") } -// CreateNamespaceIfNeeded creates a new namespace if it does not exist. -func CreateNamespaceIfNeeded(t *testing.T, client *Client, namespace string) { - _, err := client.Kube.CoreV1().Namespaces().Get(context.Background(), namespace, metav1.GetOptions{}) - - if err != nil && apierrs.IsNotFound(err) { - nsSpec := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} - _, err = client.Kube.CoreV1().Namespaces().Create(context.Background(), nsSpec, metav1.CreateOptions{}) - - if err != nil { - t.Fatalf("Failed to create Namespace: %s; %v", namespace, err) - } - - // https://github.com/kubernetes/kubernetes/issues/66689 - // We can only start creating pods after the default ServiceAccount is created by the kube-controller-manager. - err = waitForServiceAccountExists(client, "default", namespace) - if err != nil { - t.Fatal("The default ServiceAccount was not created for the Namespace:", namespace) - } +// SetupServiceAccountInClientNamespace creates a new namespace if it does not exist. +func SetupServiceAccountInClientNamespace(t *testing.T, client *Client) { + // https://github.com/kubernetes/kubernetes/issues/66689 + // We can only start creating pods after the default ServiceAccount is created by the kube-controller-manager. + err := waitForServiceAccountExists(client, "default", client.Namespace) + if err != nil { + t.Fatal("The default ServiceAccount was not created for the Namespace:", client.Namespace) + } - // If the "default" Namespace has a secret called - // "kn-eventing-test-pull-secret" then use that as the ImagePullSecret - // on the "default" ServiceAccount in this new Namespace. - // This is needed for cases where the images are in a private registry. - _, err := utils.CopySecret(client.Kube.CoreV1(), "default", testPullSecretName, namespace, "default") - if err != nil && !apierrs.IsNotFound(err) { - t.Fatalf("error copying the secret into ns %q: %s", namespace, err) - } + // If the "default" Namespace has a secret called + // "kn-eventing-test-pull-secret" then use that as the ImagePullSecret + // on the "default" ServiceAccount in this new Namespace. + // This is needed for cases where the images are in a private registry. + _, err = utils.CopySecret(client.Kube.CoreV1(), "default", testPullSecretName, client.Namespace, "default") + if err != nil && !apierrs.IsNotFound(err) { + t.Fatalf("error copying the secret into ns %q: %s", client.Namespace, err) } } From 176520fda7b897990db1d50cfc221904589a0f51 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Thu, 6 May 2021 12:12:01 +0200 Subject: [PATCH 02/10] ReuseNamespace flag --- test/e2e_flags.go | 3 +++ test/flags/eventing_environment.go | 1 + test/lib/test_runner.go | 42 +++++++++++++++++++++--------- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/test/e2e_flags.go b/test/e2e_flags.go index 0abd69d85c0..b6dc3453201 100644 --- a/test/e2e_flags.go +++ b/test/e2e_flags.go @@ -60,6 +60,9 @@ func InitializeEventingFlags() { flag.Var(&EventingFlags.Brokers, "brokers", BrokerUsage) flag.StringVar(&EventingFlags.BrokerName, "brokername", "", BrokerNameUsage) flag.StringVar(&EventingFlags.BrokerNamespace, "brokernamespace", "", BrokerNamespaceUsage) + // Might be useful in restricted environments where namespaces need to be + // created by a user with increased privileges (admin). + flag.BoolVar(&EventingFlags.ReuseNamespace, "reusenamespace", false, "Whether to re-use namespace for a test if it already exists.") flag.Parse() // If no channel is passed through the flag, initialize it as the DefaultChannel. diff --git a/test/flags/eventing_environment.go b/test/flags/eventing_environment.go index 322060d538e..83e5efd84ce 100644 --- a/test/flags/eventing_environment.go +++ b/test/flags/eventing_environment.go @@ -26,4 +26,5 @@ type EventingEnvironmentFlags struct { ReadyFile string BrokerName string BrokerNamespace string + ReuseNamespace bool } diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index df43951cc45..727e1846597 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "knative.dev/eventing/test" "os" "path/filepath" "sort" @@ -47,7 +48,7 @@ import ( const ( podLogsDir = "pod-logs" testPullSecretName = "kn-eventing-test-pull-secret" - MaxNamespaces = 20 + MaxNamespaceSkip = 20 MaxRetries = 10 RetrySleepDuration = 5 * time.Second ) @@ -161,8 +162,11 @@ func Setup(t *testing.T, runInParallel bool, options ...SetupClientOption) *Clie if err != nil { t.Fatal("Couldn't initialize clients:", err) } - - SetupServiceAccountInClientNamespace(t, client) + SetupServiceAccount(t, client) + // If namespaces are re-used the pull-secret is supposed to be created in advance. + if !test.EventingFlags.ReuseNamespace { + SetupPullSecret(t, client) + } // Run the test case in parallel if needed. if runInParallel { @@ -182,8 +186,9 @@ func Setup(t *testing.T, runInParallel bool, options ...SetupClientOption) *Clie func CreateNamespacedClient(t *testing.T) (*Client, error) { ns := "" - // Try namespaces from pre-defined range before giving up. - for i := 0; i < MaxNamespaces; i++ { + // Try next MaxNamespaceSkip namespaces before giving up. This should address the issue with + // development cycles when namespaces from previous runs were not cleaned properly. + for i := 0; i < MaxNamespaceSkip; i++ { ns = NextNamespace() client, err := NewClient( pkgTest.Flags.Kubeconfig, @@ -193,15 +198,23 @@ func CreateNamespacedClient(t *testing.T) (*Client, error) { if err != nil { return nil, err } - if err := CreateNamespace(client, ns); err != nil { - if apierrs.IsAlreadyExists(err) { - continue + if test.EventingFlags.ReuseNamespace { + // Re-using existing namespace, no need to create it. + // The namespace is supposed to be created in advance. + return client, nil + } else { + // The test is supposed to create a new test namespace for itself. + // Keep trying until we find a namespace that doesn't exist yet. + if err := CreateNamespace(client, ns); err != nil { + if apierrs.IsAlreadyExists(err) { + continue + } + return nil, err } - return nil, err } return client, nil } - return nil, errors.New("unable to find available namespace in predefined range") + return nil, errors.New("unable to find available namespace") } // NextNamespace returns the next unique namespace. @@ -318,20 +331,23 @@ func formatEvent(e *corev1.Event) string { }, "\n") } -// SetupServiceAccountInClientNamespace creates a new namespace if it does not exist. -func SetupServiceAccountInClientNamespace(t *testing.T, client *Client) { +// SetupServiceAccount creates a new namespace if it does not exist. +func SetupServiceAccount(t *testing.T, client *Client) { // https://github.com/kubernetes/kubernetes/issues/66689 // We can only start creating pods after the default ServiceAccount is created by the kube-controller-manager. err := waitForServiceAccountExists(client, "default", client.Namespace) if err != nil { t.Fatal("The default ServiceAccount was not created for the Namespace:", client.Namespace) } +} +// SetupPullSecret sets up kn-eventing-test-pull-secret on the client namespace. +func SetupPullSecret(t *testing.T, client *Client) { // If the "default" Namespace has a secret called // "kn-eventing-test-pull-secret" then use that as the ImagePullSecret // on the "default" ServiceAccount in this new Namespace. // This is needed for cases where the images are in a private registry. - _, err = utils.CopySecret(client.Kube.CoreV1(), "default", testPullSecretName, client.Namespace, "default") + _, err := utils.CopySecret(client.Kube.CoreV1(), "default", testPullSecretName, client.Namespace, "default") if err != nil && !apierrs.IsNotFound(err) { t.Fatalf("error copying the secret into ns %q: %s", client.Namespace, err) } From fdbd289fc574849b31f579c1736de15dfc1ba073 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 11 May 2021 13:44:03 +0200 Subject: [PATCH 03/10] ReuseNamespace variable in lib package * prevents import cycle when flags are accessed directly from the lib package --- test/conformance/main_test.go | 1 + test/e2e/main_test.go | 1 + test/lib/test_runner.go | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/conformance/main_test.go b/test/conformance/main_test.go index 130401ab968..b08d5e105f7 100644 --- a/test/conformance/main_test.go +++ b/test/conformance/main_test.go @@ -47,6 +47,7 @@ var brokerClass string func TestMain(m *testing.M) { os.Exit(func() int { test.InitializeEventingFlags() + testlib.ReuseNamespace = test.EventingFlags.ReuseNamespace channelTestRunner = testlib.ComponentsTestRunner{ ComponentFeatureMap: testlib.ChannelFeatureMap, ComponentsToTest: test.EventingFlags.Channels, diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index b5e26aacd1f..fd8b9f42e04 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -33,6 +33,7 @@ var brokerClass string func TestMain(m *testing.M) { test.InitializeEventingFlags() + testlib.ReuseNamespace = test.EventingFlags.ReuseNamespace channelTestRunner = testlib.ComponentsTestRunner{ ComponentFeatureMap: testlib.ChannelFeatureMap, ComponentsToTest: test.EventingFlags.Channels, diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index 727e1846597..035f4154513 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -57,6 +57,7 @@ var ( nsMutex sync.Mutex serviceCount int namespaceCount int + ReuseNamespace bool ) // ComponentsTestRunner is used to run tests against different eventing components. @@ -162,9 +163,10 @@ func Setup(t *testing.T, runInParallel bool, options ...SetupClientOption) *Clie if err != nil { t.Fatal("Couldn't initialize clients:", err) } - SetupServiceAccount(t, client) + // If namespaces are re-used the pull-secret is supposed to be created in advance. - if !test.EventingFlags.ReuseNamespace { + if !ReuseNamespace { + SetupServiceAccount(t, client) SetupPullSecret(t, client) } From 25c0d4fd62d5f05b63763e06f938f028c13f2274 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 11 May 2021 13:45:54 +0200 Subject: [PATCH 04/10] Fix gofmt --- test/lib/test_runner.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index 035f4154513..c849689c236 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -54,8 +54,8 @@ const ( ) var ( - nsMutex sync.Mutex - serviceCount int + nsMutex sync.Mutex + serviceCount int namespaceCount int ReuseNamespace bool ) @@ -193,10 +193,10 @@ func CreateNamespacedClient(t *testing.T) (*Client, error) { for i := 0; i < MaxNamespaceSkip; i++ { ns = NextNamespace() client, err := NewClient( - pkgTest.Flags.Kubeconfig, - pkgTest.Flags.Cluster, - ns, - t) + pkgTest.Flags.Kubeconfig, + pkgTest.Flags.Cluster, + ns, + t) if err != nil { return nil, err } From 9a6753f304b8105d28df00cbe6f5ac4698dbb174 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 11 May 2021 13:47:43 +0200 Subject: [PATCH 05/10] Fix goimports --- test/lib/test_runner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index c849689c236..92860ab6b68 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "knative.dev/eventing/test" "os" "path/filepath" "sort" @@ -29,6 +28,8 @@ import ( "testing" "time" + "knative.dev/eventing/test" + corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" From 9d02423fbcd54b4df1fc149b4d511d3ab9f525fc Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 11 May 2021 13:48:47 +0200 Subject: [PATCH 06/10] Remove the last occurrence of import cycle --- test/lib/test_runner.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index 92860ab6b68..f1a883d195d 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -28,8 +28,6 @@ import ( "testing" "time" - "knative.dev/eventing/test" - corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -201,7 +199,7 @@ func CreateNamespacedClient(t *testing.T) (*Client, error) { if err != nil { return nil, err } - if test.EventingFlags.ReuseNamespace { + if ReuseNamespace { // Re-using existing namespace, no need to create it. // The namespace is supposed to be created in advance. return client, nil From 97a81b9d1289d35279c0186209f368d80ad62995 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 11 May 2021 14:13:30 +0200 Subject: [PATCH 07/10] Remove unused variable --- test/lib/test_runner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index f1a883d195d..07c9d9f497c 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -54,7 +54,6 @@ const ( var ( nsMutex sync.Mutex - serviceCount int namespaceCount int ReuseNamespace bool ) From e155667099ce8f93647c19ef43e0d6d12ab301bd Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 12 May 2021 11:40:38 +0200 Subject: [PATCH 08/10] Fix detection of NotFound api error * also make the retry count and interval shorter to speed up things --- test/lib/test_runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index 07c9d9f497c..ed2d1a103e9 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -48,8 +48,8 @@ const ( podLogsDir = "pod-logs" testPullSecretName = "kn-eventing-test-pull-secret" MaxNamespaceSkip = 20 - MaxRetries = 10 - RetrySleepDuration = 5 * time.Second + MaxRetries = 5 + RetrySleepDuration = 2 * time.Second ) var ( @@ -239,7 +239,7 @@ func GetNextNamespaceId() int { func CreateNamespace(client *Client, namespace string) error { err := createNamespaceWithRetry(client, namespace) if err != nil { - return fmt.Errorf("could not create namespace %s: %w", namespace, err) + return err } return nil } From ccd4fe4de979563877b6304ac09811fdcdf6bd84 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 12 May 2021 11:52:38 +0200 Subject: [PATCH 09/10] Delete NS only if we created it * leave previously existing namespaces there --- test/lib/test_runner.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index ed2d1a103e9..e8d3f7dd22c 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -305,8 +305,11 @@ func TearDown(client *Client) { } client.Tracker.Clean(true) - if err := DeleteNameSpace(client); err != nil { - client.T.Logf("Could not delete the namespace %q: %v", client.Namespace, err) + // If we're reusing existing namespaces leave the deletion to the creator. + if !ReuseNamespace { + if err := DeleteNameSpace(client); err != nil { + client.T.Logf("Could not delete the namespace %q: %v", client.Namespace, err) + } } } From 15c78f370ced9b824d26fe6ef058669737fe1906 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 12 May 2021 11:58:36 +0200 Subject: [PATCH 10/10] Call CreateNamespaceWithRetry directly --- test/lib/test_runner.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/test/lib/test_runner.go b/test/lib/test_runner.go index e8d3f7dd22c..09b4d052954 100644 --- a/test/lib/test_runner.go +++ b/test/lib/test_runner.go @@ -205,7 +205,7 @@ func CreateNamespacedClient(t *testing.T) (*Client, error) { } else { // The test is supposed to create a new test namespace for itself. // Keep trying until we find a namespace that doesn't exist yet. - if err := CreateNamespace(client, ns); err != nil { + if err := CreateNamespaceWithRetry(client, ns); err != nil { if apierrs.IsAlreadyExists(err) { continue } @@ -235,16 +235,8 @@ func GetNextNamespaceId() int { return current } -// CreateNamespace creates the given namespace. -func CreateNamespace(client *Client, namespace string) error { - err := createNamespaceWithRetry(client, namespace) - if err != nil { - return err - } - return nil -} - -func createNamespaceWithRetry(client *Client, namespace string) error { +// CreateNamespaceWithRetry creates the given namespace with retries. +func CreateNamespaceWithRetry(client *Client, namespace string) error { var ( retries int err error