From 9a4c434d6a07ef511559bf9eeb3e0c428e9f4329 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 27 Sep 2018 14:36:05 -0700 Subject: [PATCH 1/6] commands,pkg/test: support single namespace mode for local test This commit brings support for running the tests in a single namespace when using `test local`, which was previously only used by `test cluster` (where it was a requirement to properly function) --- .travis.yml | 4 ++++ commands/operator-sdk/cmd/test/local.go | 12 +++++++++++- hack/ci/setup-openshift.sh | 3 +++ pkg/test/framework.go | 10 ++++------ pkg/test/main_entry.go | 6 ++++-- pkg/test/resource_creator.go | 2 +- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index dcf6ba8e4e..7ebe32c2b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,10 @@ script: - operator-sdk test local . # test operator-sdk test flags - operator-sdk test local . --global-manifest deploy/crd.yaml --namespaced-manifest deploy/namespace-init.yaml --go-test-flags "-parallel 1" --kubeconfig $HOME/.kube/config +# test operator-sdk test local single namespace mode +- kubectl create namespace test-memcached +- operator-sdk test local . --namespace=test-memcached +- kubectl delete namespace test-memcached # go back to project root - cd ../.. - go vet ./... diff --git a/commands/operator-sdk/cmd/test/local.go b/commands/operator-sdk/cmd/test/local.go index 14191ba2f4..45b61f084f 100644 --- a/commands/operator-sdk/cmd/test/local.go +++ b/commands/operator-sdk/cmd/test/local.go @@ -33,6 +33,7 @@ type testLocalConfig struct { globalManPath string namespacedManPath string goTestFlags string + namespace string } var tlConfig testLocalConfig @@ -52,6 +53,7 @@ func NewTestLocalCmd() *cobra.Command { testCmd.Flags().StringVar(&tlConfig.globalManPath, "global-manifest", "deploy/crd.yaml", "Path to manifest for Global resources (e.g. CRD manifest)") testCmd.Flags().StringVar(&tlConfig.namespacedManPath, "namespaced-manifest", "", "Path to manifest for per-test, namespaced resources (e.g. RBAC and Operator manifest)") testCmd.Flags().StringVar(&tlConfig.goTestFlags, "go-test-flags", "", "Additional flags to pass to go test") + testCmd.Flags().StringVar(&tlConfig.namespace, "namespace", "", "If non-empty, single namespace to run tests in") return testCmd } @@ -96,8 +98,16 @@ func testLocalFunc(cmd *cobra.Command, args []string) { testArgs = append(testArgs, "-"+test.NamespacedManPathFlag, tlConfig.namespacedManPath) testArgs = append(testArgs, "-"+test.GlobalManPathFlag, tlConfig.globalManPath) testArgs = append(testArgs, "-"+test.ProjRootFlag, mustGetwd()) - testArgs = append(testArgs, strings.Split(tlConfig.goTestFlags, " ")...) + // if we do the append using an empty go flags, it inserts an empty arg, which causes + // any later flags to be ignored + if tlConfig.goTestFlags != "" { + testArgs = append(testArgs, strings.Split(tlConfig.goTestFlags, " ")...) + } + if tlConfig.namespace != "" { + testArgs = append(testArgs, "-"+test.SingleNamespaceFlag, "-parallel=1") + } dc := exec.Command("go", testArgs...) + dc.Env = append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace)) dc.Dir = mustGetwd() dc.Stdout = os.Stdout dc.Stderr = os.Stderr diff --git a/hack/ci/setup-openshift.sh b/hack/ci/setup-openshift.sh index 39425c50d0..843b6a9b94 100755 --- a/hack/ci/setup-openshift.sh +++ b/hack/ci/setup-openshift.sh @@ -10,3 +10,6 @@ tar xvzOf oc.tar.gz openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit/oc oc cluster up # Become cluster admin oc login -u system:admin + +# kubectl is needed for the single namespace local test +curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.10.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ diff --git a/pkg/test/framework.go b/pkg/test/framework.go index 6e8000ff9c..5045854801 100644 --- a/pkg/test/framework.go +++ b/pkg/test/framework.go @@ -52,13 +52,12 @@ type Framework struct { DynamicClient dynclient.Client DynamicDecoder runtime.Decoder NamespacedManPath *string - InCluster bool + SingleNamespace *bool } -func setup(kubeconfigPath, namespacedManPath *string) error { +func setup(kubeconfigPath, namespacedManPath *string, singleNamespace *bool) error { var err error var kubeconfig *rest.Config - inCluster := false if *kubeconfigPath == "incluster" { // Work around https://github.com/kubernetes/kubernetes/issues/40973 if len(os.Getenv("KUBERNETES_SERVICE_HOST")) == 0 { @@ -72,7 +71,6 @@ func setup(kubeconfigPath, namespacedManPath *string) error { os.Setenv("KUBERNETES_SERVICE_PORT", "443") } kubeconfig, err = rest.InClusterConfig() - inCluster = true } else { kubeconfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfigPath) } @@ -103,7 +101,7 @@ func setup(kubeconfigPath, namespacedManPath *string) error { DynamicClient: dynClient, DynamicDecoder: dynDec, NamespacedManPath: namespacedManPath, - InCluster: inCluster, + SingleNamespace: singleNamespace, } return nil } @@ -136,7 +134,7 @@ func AddToFrameworkScheme(addToScheme addToSchemeFunc, obj runtime.Object) error Global.RestMapper.Reset() Global.DynamicClient, err = dynclient.New(Global.KubeConfig, dynclient.Options{Scheme: Global.Scheme, Mapper: Global.RestMapper}) err = wait.PollImmediate(time.Second, time.Second*10, func() (done bool, err error) { - if Global.InCluster { + if *Global.SingleNamespace { err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: os.Getenv(TestNamespaceEnv)}, obj) } else { err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: "default"}, obj) diff --git a/pkg/test/main_entry.go b/pkg/test/main_entry.go index 608df58eb3..4e39032cb2 100644 --- a/pkg/test/main_entry.go +++ b/pkg/test/main_entry.go @@ -27,6 +27,7 @@ const ( KubeConfigFlag = "kubeconfig" NamespacedManPathFlag = "namespacedMan" GlobalManPathFlag = "globalMan" + SingleNamespaceFlag = "singleNamespace" TestNamespaceEnv = "TEST_NAMESPACE" ) @@ -35,13 +36,14 @@ func MainEntry(m *testing.M) { kubeconfigPath := flag.String(KubeConfigFlag, "", "path to kubeconfig") globalManPath := flag.String(GlobalManPathFlag, "", "path to operator manifest") namespacedManPath := flag.String(NamespacedManPathFlag, "", "path to rbac manifest") + singleNamespace := flag.Bool(SingleNamespaceFlag, false, "enable single namespace mode") flag.Parse() // go test always runs from the test directory; change to project root err := os.Chdir(*projRoot) if err != nil { log.Fatalf("failed to change directory to project root: %v", err) } - if err := setup(kubeconfigPath, namespacedManPath); err != nil { + if err := setup(kubeconfigPath, namespacedManPath, singleNamespace); err != nil { log.Fatalf("failed to set up framework: %v", err) } // setup context to use when setting up crd @@ -54,7 +56,7 @@ func MainEntry(m *testing.M) { os.Exit(exitCode) }() // create crd - if !Global.InCluster { + if *kubeconfigPath != "incluster" { globalYAML, err := ioutil.ReadFile(*globalManPath) if err != nil { log.Fatalf("failed to read global resource manifest: %v", err) diff --git a/pkg/test/resource_creator.go b/pkg/test/resource_creator.go index 940f603174..7d0739ba72 100644 --- a/pkg/test/resource_creator.go +++ b/pkg/test/resource_creator.go @@ -31,7 +31,7 @@ func (ctx *TestCtx) GetNamespace() (string, error) { if ctx.Namespace != "" { return ctx.Namespace, nil } - if Global.InCluster { + if *Global.SingleNamespace { ctx.Namespace = os.Getenv(TestNamespaceEnv) return ctx.Namespace, nil } From ae8048aa75a45dcaa883bbe9f2b263047542563d Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 27 Sep 2018 15:45:38 -0700 Subject: [PATCH 2/6] pkg/test/framework.go: enable singleNamespace for incluster --- pkg/test/framework.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/test/framework.go b/pkg/test/framework.go index 5045854801..eb93457cf7 100644 --- a/pkg/test/framework.go +++ b/pkg/test/framework.go @@ -71,6 +71,7 @@ func setup(kubeconfigPath, namespacedManPath *string, singleNamespace *bool) err os.Setenv("KUBERNETES_SERVICE_PORT", "443") } kubeconfig, err = rest.InClusterConfig() + *singleNamespace = true } else { kubeconfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfigPath) } From 6ce37fe0668b58ff94b4b77323ed8f863bc9c32e Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Thu, 27 Sep 2018 16:22:22 -0700 Subject: [PATCH 3/6] doc: update docs with new test local --namespace flag --- doc/sdk-cli-reference.md | 1 + doc/test-framework/writing-e2e-tests.md | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/doc/sdk-cli-reference.md b/doc/sdk-cli-reference.md index f61b4f87b6..6ef5eac68d 100644 --- a/doc/sdk-cli-reference.md +++ b/doc/sdk-cli-reference.md @@ -175,6 +175,7 @@ Runs the tests locally * `--kubeconfig` string - location of kubeconfig for kubernetes cluster (default "~/.kube/config") * `--global-manifest` string - path to manifest for global resources (default "deploy/crd.yaml) * `--namespaced-manifest` string - path to manifest for per-test, namespaced resources (default: combines deploy/sa.yaml, deploy/rbac.yaml, and deploy/operator.yaml) +* `--namespace` string - if non-empty, single namespace to run tests in (e.g. "operator-test") (default: "") * `--go-test-flags` string - extra arguments to pass to `go test` (e.g. -f "-v -parallel=2") * `-h, --help` - help for local diff --git a/doc/test-framework/writing-e2e-tests.md b/doc/test-framework/writing-e2e-tests.md index a08e4356c2..3c343adeda 100644 --- a/doc/test-framework/writing-e2e-tests.md +++ b/doc/test-framework/writing-e2e-tests.md @@ -207,6 +207,13 @@ as an argument. You can use `--help` to view the other configuration options and $ operator-sdk test local ./test/e2e --go-test-flags "-v -parallel=2" ``` +If you wish to run all the tests in 1 namespace (which also forces `-parallel=1`), you can use the `--namespace` flag: + +```shell +$ kubectl create namespace operator-test +$ operator-sdk test local ./test/e2e --namespace operator-test +``` + For more documentation on the `operator-sdk test local` command, see the [SDK CLI Reference][sdk-cli-ref] doc. For advanced use cases, it is possible to run the tests via `go test` directly. As long as all flags defined @@ -258,6 +265,7 @@ Test Successfully Completed ``` The `test cluster` command will deploy a test pod in the given namespace that will run the e2e tests packaged in the image. +The tests run sequentially in the namespace (`-parallel=1`), the same as running `operator-sdk test local --namespace `. The command will wait until the tests succeed (pod phase=`Succeeded`) or fail (pod phase=`Failed`). If the tests fail, the command will output the test pod logs which should be the standard go test error logs. From 4c09f6554b9b46a18e7bba7eb90f9cfd7f6d0d10 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Fri, 28 Sep 2018 15:20:21 -0700 Subject: [PATCH 4/6] pkg/test/resource_creator.go: check namespace env value --- pkg/test/resource_creator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/test/resource_creator.go b/pkg/test/resource_creator.go index 7d0739ba72..861432a22b 100644 --- a/pkg/test/resource_creator.go +++ b/pkg/test/resource_creator.go @@ -33,6 +33,9 @@ func (ctx *TestCtx) GetNamespace() (string, error) { } if *Global.SingleNamespace { ctx.Namespace = os.Getenv(TestNamespaceEnv) + if len(ctx.Namespace) == 0 { + return "", fmt.Errorf("namepspace set in %s cannot be empty", TestNamespaceEnv) + } return ctx.Namespace, nil } // create namespace From da8f2437b2b1b576a05b59775b249401bb2c7043 Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Fri, 28 Sep 2018 15:30:16 -0700 Subject: [PATCH 5/6] pkg/test/resource_creator.go: fix typo --- pkg/test/resource_creator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/test/resource_creator.go b/pkg/test/resource_creator.go index 861432a22b..57bf96e9e4 100644 --- a/pkg/test/resource_creator.go +++ b/pkg/test/resource_creator.go @@ -34,7 +34,7 @@ func (ctx *TestCtx) GetNamespace() (string, error) { if *Global.SingleNamespace { ctx.Namespace = os.Getenv(TestNamespaceEnv) if len(ctx.Namespace) == 0 { - return "", fmt.Errorf("namepspace set in %s cannot be empty", TestNamespaceEnv) + return "", fmt.Errorf("namespace set in %s cannot be empty", TestNamespaceEnv) } return ctx.Namespace, nil } From e69c886e64540a216954069388054cf55c68ac1f Mon Sep 17 00:00:00 2001 From: Alexander Pavel Date: Fri, 28 Sep 2018 15:48:27 -0700 Subject: [PATCH 6/6] pkg/test: change handling of TestNamespaceEnv --- pkg/test/framework.go | 17 +++++++++++++---- pkg/test/resource_creator.go | 8 ++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/test/framework.go b/pkg/test/framework.go index eb93457cf7..937667ca30 100644 --- a/pkg/test/framework.go +++ b/pkg/test/framework.go @@ -52,7 +52,8 @@ type Framework struct { DynamicClient dynclient.Client DynamicDecoder runtime.Decoder NamespacedManPath *string - SingleNamespace *bool + SingleNamespace bool + Namespace string } func setup(kubeconfigPath, namespacedManPath *string, singleNamespace *bool) error { @@ -94,6 +95,13 @@ func setup(kubeconfigPath, namespacedManPath *string, singleNamespace *bool) err return fmt.Errorf("failed to build the dynamic client: %v", err) } dynDec := serializer.NewCodecFactory(scheme).UniversalDeserializer() + namespace := "" + if *singleNamespace { + namespace = os.Getenv(TestNamespaceEnv) + if len(namespace) == 0 { + return fmt.Errorf("namespace set in %s cannot be empty", TestNamespaceEnv) + } + } Global = &Framework{ KubeConfig: kubeconfig, KubeClient: kubeclient, @@ -102,7 +110,8 @@ func setup(kubeconfigPath, namespacedManPath *string, singleNamespace *bool) err DynamicClient: dynClient, DynamicDecoder: dynDec, NamespacedManPath: namespacedManPath, - SingleNamespace: singleNamespace, + SingleNamespace: *singleNamespace, + Namespace: namespace, } return nil } @@ -135,8 +144,8 @@ func AddToFrameworkScheme(addToScheme addToSchemeFunc, obj runtime.Object) error Global.RestMapper.Reset() Global.DynamicClient, err = dynclient.New(Global.KubeConfig, dynclient.Options{Scheme: Global.Scheme, Mapper: Global.RestMapper}) err = wait.PollImmediate(time.Second, time.Second*10, func() (done bool, err error) { - if *Global.SingleNamespace { - err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: os.Getenv(TestNamespaceEnv)}, obj) + if Global.SingleNamespace { + err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: Global.Namespace}, obj) } else { err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: "default"}, obj) } diff --git a/pkg/test/resource_creator.go b/pkg/test/resource_creator.go index 57bf96e9e4..3a693a9321 100644 --- a/pkg/test/resource_creator.go +++ b/pkg/test/resource_creator.go @@ -19,7 +19,6 @@ import ( goctx "context" "fmt" "io/ioutil" - "os" yaml "gopkg.in/yaml.v2" core "k8s.io/api/core/v1" @@ -31,11 +30,8 @@ func (ctx *TestCtx) GetNamespace() (string, error) { if ctx.Namespace != "" { return ctx.Namespace, nil } - if *Global.SingleNamespace { - ctx.Namespace = os.Getenv(TestNamespaceEnv) - if len(ctx.Namespace) == 0 { - return "", fmt.Errorf("namespace set in %s cannot be empty", TestNamespaceEnv) - } + if Global.SingleNamespace { + ctx.Namespace = Global.Namespace return ctx.Namespace, nil } // create namespace