diff --git a/hack/tests/e2e-go.sh b/hack/tests/e2e-go.sh index f07420b4b7..4d13723953 100755 --- a/hack/tests/e2e-go.sh +++ b/hack/tests/e2e-go.sh @@ -8,14 +8,11 @@ source ./hack/lib/test_lib.sh source ./hack/lib/image_lib.sh test_dir=./test -tests=$test_dir/e2e +tests=$test_dir/e2e-go export TRACE=1 export GO111MODULE=on setup_envs $tmp_sdk_root -docker pull gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 -load_image_if_kind gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0 - -go test -v $tests +go test $tests -v -ginkgo.v diff --git a/test/e2e-ansible/e2e_ansible_suite_test.go b/test/e2e-ansible/e2e_ansible_suite_test.go index 2e63eecae0..8946630458 100644 --- a/test/e2e-ansible/e2e_ansible_suite_test.go +++ b/test/e2e-ansible/e2e_ansible_suite_test.go @@ -38,8 +38,8 @@ func TestE2EAnsible(t *testing.T) { var ( tc testutils.TestContext - // isPromethuesManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus - isPromethuesManagedBySuite = true + // isPrometheusManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus + isPrometheusManagedBySuite = true // isOLMManagedBySuite is true when the suite tests is installing/uninstalling the OLM isOLMManagedBySuite = true // kubectx stores the k8s context from where the tests are running @@ -66,13 +66,13 @@ var _ = BeforeSuite(func(done Done) { output, err := tc.Kubectl.Command("api-resources") Expect(err).NotTo(HaveOccurred()) if strings.Contains(output, "servicemonitors") { - isPromethuesManagedBySuite = false + isPrometheusManagedBySuite = false } if strings.Contains(output, "clusterserviceversions") { isOLMManagedBySuite = false } - if isPromethuesManagedBySuite { + if isPrometheusManagedBySuite { By("installing Prometheus") Expect(tc.InstallPrometheusOperManager()).To(Succeed()) @@ -176,7 +176,7 @@ var _ = BeforeSuite(func(done Done) { // AfterSuite run after all the specs have run, regardless of whether any tests have failed to ensures that // all be cleaned up var _ = AfterSuite(func() { - if isPromethuesManagedBySuite { + if isPrometheusManagedBySuite { By("uninstalling Prometheus") tc.UninstallPrometheusOperManager() } diff --git a/test/e2e-go/e2e_go_cluster_test.go b/test/e2e-go/e2e_go_cluster_test.go new file mode 100644 index 0000000000..662981cfb2 --- /dev/null +++ b/test/e2e-go/e2e_go_cluster_test.go @@ -0,0 +1,191 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Modified from https://github.com/kubernetes-sigs/kubebuilder/tree/39224f0/test/e2e/v3 + +package e2e_go_test + +import ( + "encoding/base64" + "fmt" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo" //nolint:golint + . "github.com/onsi/gomega" //nolint:golint + kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils" +) + +var _ = Describe("operator-sdk", func() { + var controllerPodName string + + Context("built with operator-sdk", func() { + BeforeEach(func() { + By("enabling Prometheus via the kustomization.yaml") + Expect(kbtestutils.UncommentCode( + filepath.Join(tc.Dir, "config", "default", "kustomization.yaml"), + "#- ../prometheus", "#")).To(Succeed()) + + By("deploying project on the cluster") + err := tc.Make("deploy", "IMG="+tc.ImageName) + Expect(err).Should(Succeed()) + }) + AfterEach(func() { + By("cleaning up the operator and resources") + defaultOutput, err := tc.KustomizeBuild(filepath.Join("config", "default")) + Expect(err).NotTo(HaveOccurred()) + _, err = tc.Kubectl.WithInput(string(defaultOutput)).Command("delete", "-f", "-") + Expect(err).NotTo(HaveOccurred()) + + By("deleting Curl Pod created") + _, _ = tc.Kubectl.Delete(true, "pod", "curl") + + By("cleaning up permissions") + _, _ = tc.Kubectl.Command("delete", "clusterrolebinding", + fmt.Sprintf("metrics-%s", tc.TestSuffix)) + + By("undeploy project") + _ = tc.Make("undeploy") + + By("ensuring that the namespace was deleted") + verifyNamespaceDeleted := func() error { + _, err := tc.Kubectl.Command("get", "namespace", tc.Kubectl.Namespace) + if strings.Contains(err.Error(), "(NotFound): namespaces") { + return err + } + return nil + } + Eventually(verifyNamespaceDeleted, 2*time.Minute, time.Second).ShouldNot(Succeed()) + }) + + It("should run correctly in a cluster", func() { + By("checking if the Operator project Pod is running") + verifyControllerUp := func() error { + By("getting the controller-manager pod name") + podOutput, err := tc.Kubectl.Get( + true, + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}") + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the created controller-manager Pod") + podNames := kbtestutils.GetNonEmptyLines(podOutput) + if len(podNames) != 1 { + return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) + } + controllerPodName = podNames[0] + Expect(controllerPodName).Should(ContainSubstring("controller-manager")) + + By("checking the controller-manager Pod is running") + status, err := tc.Kubectl.Get( + true, + "pods", controllerPodName, "-o", "jsonpath={.status.phase}") + Expect(err).NotTo(HaveOccurred()) + if status != "Running" { + return fmt.Errorf("controller pod in %s status", status) + } + return nil + } + Eventually(verifyControllerUp, 2*time.Minute, time.Second).Should(Succeed()) + + By("ensuring the created ServiceMonitor for the manager") + _, err := tc.Kubectl.Get( + true, + "ServiceMonitor", + fmt.Sprintf("e2e-%s-controller-manager-metrics-monitor", tc.TestSuffix)) + Expect(err).NotTo(HaveOccurred()) + + By("ensuring the created metrics Service for the manager") + _, err = tc.Kubectl.Get( + true, + "Service", + fmt.Sprintf("e2e-%s-controller-manager-metrics-service", tc.TestSuffix)) + Expect(err).NotTo(HaveOccurred()) + + By("creating an instance of CR") + // currently controller-runtime doesn't provide a readiness probe, we retry a few times + // we can change it to probe the readiness endpoint after CR supports it. + sampleFile := filepath.Join("config", "samples", + fmt.Sprintf("%s_%s_%s.yaml", tc.Group, tc.Version, strings.ToLower(tc.Kind))) + Eventually(func() error { + _, err = tc.Kubectl.Apply(true, "-f", sampleFile) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("ensuring the created resource object gets reconciled in controller") + managerContainerLogs := func() string { + logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(managerContainerLogs, time.Minute, time.Second).Should(ContainSubstring("Successfully Reconciled")) + + By("granting permissions to access the metrics and read the token") + _, err = tc.Kubectl.Command( + "create", + "clusterrolebinding", + fmt.Sprintf("metrics-%s", tc.TestSuffix), + fmt.Sprintf("--clusterrole=e2e-%s-metrics-reader", tc.TestSuffix), + fmt.Sprintf("--serviceaccount=%s:default", tc.Kubectl.Namespace)) + Expect(err).NotTo(HaveOccurred()) + + By("getting the token") + b64Token, err := tc.Kubectl.Get( + true, + "secrets", + "-o=jsonpath={.items[0].data.token}") + Expect(err).NotTo(HaveOccurred()) + token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token)) + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(HaveLen(0)) + + By("creating a pod with curl image") + // todo: the flag --generator=run-pod/v1 is deprecated, however, shows that besides + // it should not make any difference and work locally successfully when the flag is removed + // travis has been failing and the curl pod is not found when the flag is not used + cmdOpts := []string{ + "run", "--generator=run-pod/v1", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--", + "curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token), + fmt.Sprintf("https://e2e-%v-controller-manager-metrics-service.e2e-%v-system.svc:8443/metrics", + tc.TestSuffix, tc.TestSuffix), + } + _, err = tc.Kubectl.CommandInNamespace(cmdOpts...) + Expect(err).NotTo(HaveOccurred()) + + By("validating the curl pod running as expected") + verifyCurlUp := func() error { + // Validate pod status + status, err := tc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + Expect(err).NotTo(HaveOccurred()) + if status != "Completed" && status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) + } + return nil + } + Eventually(verifyCurlUp, 4*time.Minute, time.Second).Should(Succeed()) + + By("checking metrics endpoint serving as expected") + getCurlLogs := func() string { + logOutput, err := tc.Kubectl.Logs("curl") + Expect(err).NotTo(HaveOccurred()) + return logOutput + } + Eventually(getCurlLogs, time.Minute, time.Second).Should(ContainSubstring("< HTTP/2 200")) + }) + }) +}) diff --git a/test/e2e-go/e2e_go_local_test.go b/test/e2e-go/e2e_go_local_test.go new file mode 100644 index 0000000000..b9839ff10a --- /dev/null +++ b/test/e2e-go/e2e_go_local_test.go @@ -0,0 +1,50 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_go_test + +import ( + "os/exec" + + . "github.com/onsi/ginkgo" //nolint:golint + . "github.com/onsi/gomega" //nolint:golint +) + +var _ = Describe("Running Go projects", func() { + Context("built with operator-sdk", func() { + + BeforeEach(func() { + By("installing CRD's") + err := tc.Make("install") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + By("uninstalling CRD's") + err := tc.Make("uninstall") + Expect(err).NotTo(HaveOccurred()) + }) + + It("should run correctly locally", func() { + By("running the project") + cmd := exec.Command("make", "run") + err := cmd.Start() + Expect(err).NotTo(HaveOccurred()) + + By("killing the project") + err = cmd.Process.Kill() + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/test/e2e-go/e2e_go_olm_test.go b/test/e2e-go/e2e_go_olm_test.go new file mode 100644 index 0000000000..f3ee4c8e0d --- /dev/null +++ b/test/e2e-go/e2e_go_olm_test.go @@ -0,0 +1,136 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_go_test + +import ( + "encoding/json" + "os/exec" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" + + testutils "github.com/operator-framework/operator-sdk/test/internal" +) + +var _ = Describe("Integrating Go Projects with OLM", func() { + Context("with operator-sdk", func() { + const operatorVersion = "0.0.1" + + const ( + OLMBundleValidationTest = "olm-bundle-validation" + OLMCRDsHaveValidationTest = "olm-crds-have-validation" + OLMCRDsHaveResourcesTest = "olm-crds-have-resources" + OLMSpecDescriptorsTest = "olm-spec-descriptors" + OLMStatusDescriptorsTest = "olm-status-descriptors" + ) + + It("should generate and run a valid OLM bundle and packagemanifests", func() { + By("generating the operator bundle") + // Turn off interactive prompts for all generation tasks. + replace := "operator-sdk generate kustomize manifests" + testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false") + err := tc.Make("bundle", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + + By("building the operator bundle image") + // Use the existing image tag but with a "-bundle" suffix. + imageSplit := strings.SplitN(tc.ImageName, ":", 2) + bundleImage := imageSplit[0] + "-bundle" + if len(imageSplit) == 2 { + bundleImage += ":" + imageSplit[1] + } + err = tc.Make("bundle-build", "BUNDLE_IMG="+bundleImage) + Expect(err).NotTo(HaveOccurred()) + + if isRunningOnKind() { + By("loading the bundle image into Kind cluster") + err = tc.LoadImageToKindClusterWithName(bundleImage) + Expect(err).Should(Succeed()) + } + + By("adding the 'packagemanifests' rule to the Makefile") + err = tc.AddPackagemanifestsTarget() + Expect(err).Should(Succeed()) + + By("generating the operator package manifests") + err = tc.Make("packagemanifests", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + + By("running the package manifests-formatted operator") + Expect(err).NotTo(HaveOccurred()) + runPkgManCmd := exec.Command(tc.BinaryName, "run", "packagemanifests", + "--install-mode", "AllNamespaces", + "--version", operatorVersion, + "--timeout", "4m") + _, err = tc.Run(runPkgManCmd) + Expect(err).NotTo(HaveOccurred()) + + By("running basic scorecard tests") + var scorecardOutput v1alpha3.TestList + runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err := tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(len(scorecardOutput.Items)).To(Equal(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + + By("running custom scorecard tests") + runScorecardCmd = exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=custom", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(len(scorecardOutput.Items)).To(Equal(2)) + + By("running olm scorecard tests") + runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=olm", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) + Expect(err).To(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + + resultTable := make(map[string]v1alpha3.State) + resultTable[OLMStatusDescriptorsTest] = v1alpha3.FailState + resultTable[OLMCRDsHaveResourcesTest] = v1alpha3.FailState + resultTable[OLMBundleValidationTest] = v1alpha3.PassState + resultTable[OLMSpecDescriptorsTest] = v1alpha3.FailState + resultTable[OLMCRDsHaveValidationTest] = v1alpha3.PassState + + Expect(len(scorecardOutput.Items)).To(Equal(len(resultTable))) + for a := 0; a < len(scorecardOutput.Items); a++ { + Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(resultTable[scorecardOutput.Items[a].Status.Results[0].Name])) + } + + By("destroying the deployed package manifests-formatted operator") + cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", projectName, + "--timeout", "4m") + _, err = tc.Run(cleanupPkgManCmd) + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/test/e2e-go/e2e_go_suite_test.go b/test/e2e-go/e2e_go_suite_test.go new file mode 100644 index 0000000000..50b208329d --- /dev/null +++ b/test/e2e-go/e2e_go_suite_test.go @@ -0,0 +1,169 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_go_test + +import ( + "fmt" + "path" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo" //nolint:golint + . "github.com/onsi/gomega" //nolint:golint + kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils" + + testutils "github.com/operator-framework/operator-sdk/test/internal" +) + +// TestE2EGo ensures the Go projects built with the SDK tool by using its binary. +func TestE2EGo(t *testing.T) { + if testing.Short() { + t.Skip("skipping Operator SDK E2E Go Suite testing in short mode") + } + RegisterFailHandler(Fail) + RunSpecs(t, "E2EGo Suite") +} + +var ( + tc testutils.TestContext + // isPrometheusManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus + isPrometheusManagedBySuite = true + // isOLMManagedBySuite is true when the suite tests is installing/uninstalling the OLM + isOLMManagedBySuite = true + // kubectx stores the k8s context from where the tests are running + kubectx string + // projectName is the name of the test project + projectName string +) + +// BeforeSuite run before any specs are run to perform the required actions for all e2e Go tests. +var _ = BeforeSuite(func(done Done) { + var err error + + By("creating a new test context") + tc, err = testutils.NewTestContext("GO111MODULE=on") + Expect(err).NotTo(HaveOccurred()) + Expect(tc.Prepare()).To(Succeed()) + projectName = filepath.Base(tc.Dir) + + By("checking the cluster type") + kubectx, err = tc.Kubectl.Command("config", "current-context") + Expect(err).Should(Succeed()) + + By("checking API resources applied on Cluster") + output, err := tc.Kubectl.Command("api-resources") + Expect(err).NotTo(HaveOccurred()) + if strings.Contains(output, "servicemonitors") { + isPrometheusManagedBySuite = false + } + if strings.Contains(output, "clusterserviceversions") { + isOLMManagedBySuite = false + } + + if isPrometheusManagedBySuite { + By("installing Prometheus") + Expect(tc.InstallPrometheusOperManager()).To(Succeed()) + + By("ensuring provisioned Prometheus Manager Service") + Eventually(func() error { + _, err := tc.Kubectl.Get( + false, + "Service", "prometheus-operator") + return err + }, 3*time.Minute, time.Second).Should(Succeed()) + } + + if isOLMManagedBySuite { + By("installing OLM") + Expect(tc.InstallOLM()).To(Succeed()) + } + + By("initializing a project") + projectName := filepath.Base(tc.Dir) + err = tc.Init( + "--project-version", "3-alpha", + "--repo", path.Join("github.com", "example", projectName), + "--domain", tc.Domain, + "--fetch-deps=false") + Expect(err).Should(Succeed()) + + By("by adding scorecard custom patch file") + err = tc.AddScorecardCustomPatchFile() + Expect(err).NotTo(HaveOccurred()) + + By("creating an API definition") + err = tc.CreateAPI( + "--group", tc.Group, + "--version", tc.Version, + "--kind", tc.Kind, + "--namespaced", + "--resource", + "--controller", + "--make=false") + Expect(err).Should(Succeed()) + + By("implementing the API") + Expect(kbtestutils.InsertCode( + filepath.Join(tc.Dir, "api", tc.Version, fmt.Sprintf("%s_types.go", strings.ToLower(tc.Kind))), + fmt.Sprintf(`type %sSpec struct { +`, tc.Kind), + ` // +optional + Count int `+"`"+`json:"count,omitempty"`+"`"+` +`)).Should(Succeed()) + + By("enabling Prometheus via the kustomization.yaml") + Expect(kbtestutils.UncommentCode( + filepath.Join(tc.Dir, "config", "default", "kustomization.yaml"), + "#- ../prometheus", "#")).To(Succeed()) + + By("checking the kustomize setup") + err = tc.Make("kustomize") + Expect(err).Should(Succeed()) + + By("building the project image") + err = tc.Make("docker-build", "IMG="+tc.ImageName) + Expect(err).Should(Succeed()) + + if isRunningOnKind() { + By("loading the project image into Kind cluster") + err = tc.LoadImageToKindCluster() + Expect(err).Should(Succeed()) + } + + close(done) +}, 360) + +// AfterSuite run after all the specs have run, regardless of whether any tests have failed to ensures that +// all be cleaned up +var _ = AfterSuite(func() { + if isPrometheusManagedBySuite { + By("uninstalling Prometheus") + tc.UninstallPrometheusOperManager() + } + if isOLMManagedBySuite { + By("uninstalling OLM") + tc.UninstallOLM() + } + + By("destroying container image and work dir") + tc.Destroy() +}) + +// isRunningOnKind returns true when the tests are executed in a Kind Cluster +func isRunningOnKind() bool { + return strings.Contains(kubectx, "kind") +} diff --git a/test/e2e-helm/e2e_helm_suite_test.go b/test/e2e-helm/e2e_helm_suite_test.go index 94d3d76b14..5fe54770be 100644 --- a/test/e2e-helm/e2e_helm_suite_test.go +++ b/test/e2e-helm/e2e_helm_suite_test.go @@ -37,8 +37,8 @@ func TestE2EHelm(t *testing.T) { var ( tc testutils.TestContext - // isPromethuesManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus - isPromethuesManagedBySuite = true + // isPrometheusManagedBySuite is true when the suite tests is installing/uninstalling the Prometheus + isPrometheusManagedBySuite = true // isOLMManagedBySuite is true when the suite tests is installing/uninstalling the OLM isOLMManagedBySuite = true // kubectx stores the k8s context from where the tests are running @@ -65,13 +65,13 @@ var _ = BeforeSuite(func(done Done) { output, err := tc.Kubectl.Command("api-resources") Expect(err).NotTo(HaveOccurred()) if strings.Contains(output, "servicemonitors") { - isPromethuesManagedBySuite = false + isPrometheusManagedBySuite = false } if strings.Contains(output, "clusterserviceversions") { isOLMManagedBySuite = false } - if isPromethuesManagedBySuite { + if isPrometheusManagedBySuite { By("installing Prometheus") Expect(tc.InstallPrometheusOperManager()).To(Succeed()) @@ -126,7 +126,7 @@ var _ = BeforeSuite(func(done Done) { // AfterSuite run after all the specs have run, regardless of whether any tests have failed to ensures that // all be cleaned up var _ = AfterSuite(func() { - if isPromethuesManagedBySuite { + if isPrometheusManagedBySuite { By("uninstalling Prometheus") tc.UninstallPrometheusOperManager() } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go deleted file mode 100644 index 1e13c222ef..0000000000 --- a/test/e2e/e2e_suite_test.go +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright 2020 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Modified from https://github.com/kubernetes-sigs/kubebuilder/tree/39224f0/test/e2e/v3 - -package e2e_test - -import ( - "bytes" - "encoding/json" - "fmt" - "os/exec" - "path" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" - kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils" - - testutils "github.com/operator-framework/operator-sdk/test/internal" -) - -const ( - OLMBundleValidationTest = "olm-bundle-validation" - OLMCRDsHaveValidationTest = "olm-crds-have-validation" - OLMCRDsHaveResourcesTest = "olm-crds-have-resources" - OLMSpecDescriptorsTest = "olm-spec-descriptors" - OLMStatusDescriptorsTest = "olm-status-descriptors" -) - -var _ = Describe("operator-sdk", func() { - Context("with the new project layout", func() { - var ( - tc testutils.TestContext - projectName string - operatorVersion = "0.0.1" - ) - - BeforeEach(func() { - By("creating a new test context") - var err error - tc, err = testutils.NewTestContext("GO111MODULE=on") - Expect(err).NotTo(HaveOccurred()) - Expect(tc.Prepare()).To(Succeed()) - projectName = filepath.Base(tc.Dir) - - By("installing OLM") - Expect(tc.InstallOLM()).To(Succeed()) - }) - - AfterEach(func() { - By("cleaning up created API objects during test process") - tc.CleanupManifests(filepath.Join("config", "default")) - - By("removing container image and work dir") - tc.Destroy() - - By("uninstalling OLM") - tc.UninstallOLM() - }) - - It("should generate a runnable project", func() { - var controllerPodName string - By("initializing a project") - err := tc.Init( - "--project-version", "3-alpha", - "--repo", path.Join("github.com", "example", projectName), - "--domain", tc.Domain, - "--fetch-deps=false") - Expect(err).Should(Succeed()) - err = tc.AddScorecardCustomPatchFile() - Expect(err).NotTo(HaveOccurred()) - - By("creating an API definition") - err = tc.CreateAPI( - "--group", tc.Group, - "--version", tc.Version, - "--kind", tc.Kind, - "--namespaced", - "--resource", - "--controller", - "--make=false") - Expect(err).Should(Succeed()) - - By("implementing the API") - Expect(kbtestutils.InsertCode( - filepath.Join(tc.Dir, "api", tc.Version, fmt.Sprintf("%s_types.go", strings.ToLower(tc.Kind))), - fmt.Sprintf(`type %sSpec struct { -`, tc.Kind), - ` // +optional - Count int `+"`"+`json:"count,omitempty"`+"`"+` -`)).Should(Succeed()) - - By("building the operator image") - err = tc.Make("docker-build", "IMG="+tc.ImageName) - Expect(err).Should(Succeed()) - - kubectx, err := tc.Kubectl.Command("config", "current-context") - Expect(err).Should(Succeed()) - - if strings.Contains(kubectx, "kind") { - By("loading the operator image into the test cluster") - err = tc.LoadImageToKindCluster() - Expect(err).Should(Succeed()) - } - - By("deploying the controller manager") - err = tc.Make("deploy", "IMG="+tc.ImageName) - Expect(err).Should(Succeed()) - - By("ensuring the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - podOutput, err := tc.Kubectl.Get( - true, - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}") - Expect(err).NotTo(HaveOccurred()) - podNames := kbtestutils.GetNonEmptyLines(podOutput) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } - controllerPodName = podNames[0] - Expect(controllerPodName).Should(ContainSubstring("controller-manager")) - - // Validate pod status - status, err := tc.Kubectl.Get( - true, - "pods", controllerPodName, "-o", "jsonpath={.status.phase}") - Expect(err).NotTo(HaveOccurred()) - if status != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil - } - Eventually(verifyControllerUp, time.Minute, time.Second).Should(Succeed()) - - By("creating an instance of CR") - // currently controller-runtime doesn't provide a readiness probe, we retry a few times - // we can change it to probe the readiness endpoint after CR supports it. - sampleFile := filepath.Join("config", "samples", - fmt.Sprintf("%s_%s_%s.yaml", tc.Group, tc.Version, strings.ToLower(tc.Kind))) - Eventually(func() error { - _, err = tc.Kubectl.Apply(true, "-f", sampleFile) - return err - }, time.Minute, time.Second).Should(Succeed()) - - By("ensuring the created resource object gets reconciled in controller") - managerContainerLogs := func() string { - logOutput, err := tc.Kubectl.Logs(controllerPodName, "-c", "manager") - Expect(err).NotTo(HaveOccurred()) - return logOutput - } - Eventually(managerContainerLogs, time.Minute, time.Second).Should(ContainSubstring("Successfully Reconciled")) - - By("cleaning up the operator and resources") - defaultOutput, err := tc.KustomizeBuild(filepath.Join("config", "default")) - Expect(err).NotTo(HaveOccurred()) - _, err = tc.Kubectl.WithInput(string(defaultOutput)).Command("delete", "-f", "-") - Expect(err).NotTo(HaveOccurred()) - - By("generating the operator bundle") - // Turn off interactive prompts for all generation tasks. - replace := "operator-sdk generate kustomize manifests" - testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false") - err = tc.Make("bundle", "IMG="+tc.ImageName) - Expect(err).NotTo(HaveOccurred()) - - By("building the operator bundle image") - // Use the existing image tag but with a "-bundle" suffix. - imageSplit := strings.SplitN(tc.ImageName, ":", 2) - bundleImage := imageSplit[0] + "-bundle" - if len(imageSplit) == 2 { - bundleImage += ":" + imageSplit[1] - } - err = tc.Make("bundle-build", "BUNDLE_IMG="+bundleImage) - Expect(err).NotTo(HaveOccurred()) - - By("generating the operator package manifests") - Expect(tc.Make("manifests")).Should(Succeed()) - genKustomizeCmd := exec.Command(tc.BinaryName, "generate", "kustomize", "manifests") - _, err = tc.Run(genKustomizeCmd) - Expect(err).NotTo(HaveOccurred()) - manifestsOutput, err := tc.KustomizeBuild(filepath.Join("config", "manifests")) - Expect(err).NotTo(HaveOccurred()) - genPkgManCmd := exec.Command(tc.BinaryName, "generate", "packagemanifests", "--version", "0.0.1") - tc.Stdin = bytes.NewBuffer(manifestsOutput) - _, err = tc.Run(genPkgManCmd) - Expect(err).NotTo(HaveOccurred()) - - By("running the package manifests-formatted operator") - _, err = tc.Kubectl.Command("create", "namespace", tc.Kubectl.Namespace) - Expect(err).NotTo(HaveOccurred()) - runPkgManCmd := exec.Command(tc.BinaryName, "run", "packagemanifests", - "--install-mode", "AllNamespaces", - "--namespace", tc.Kubectl.Namespace, - "--version", operatorVersion, - "--timeout", "4m") - _, err = tc.Run(runPkgManCmd) - Expect(err).NotTo(HaveOccurred()) - - By("running basic scorecard tests") - var scorecardOutput v1alpha3.TestList - runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=basic", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err := tc.Run(runScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - Expect(len(scorecardOutput.Items)).To(Equal(1)) - Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) - - By("running custom scorecard tests") - runScorecardCmd = exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=custom", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err = tc.Run(runScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - Expect(len(scorecardOutput.Items)).To(Equal(2)) - - By("running olm scorecard tests") - runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=olm", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) - Expect(err).To(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - - resultTable := make(map[string]v1alpha3.State) - resultTable[OLMStatusDescriptorsTest] = v1alpha3.FailState - resultTable[OLMCRDsHaveResourcesTest] = v1alpha3.FailState - resultTable[OLMBundleValidationTest] = v1alpha3.PassState - resultTable[OLMSpecDescriptorsTest] = v1alpha3.FailState - resultTable[OLMCRDsHaveValidationTest] = v1alpha3.PassState - - Expect(len(scorecardOutput.Items)).To(Equal(len(resultTable))) - for a := 0; a < len(scorecardOutput.Items); a++ { - Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(resultTable[scorecardOutput.Items[a].Status.Results[0].Name])) - } - - By("destroying the deployed package manifests-formatted operator") - cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", projectName, - "--namespace", tc.Kubectl.Namespace, - "--timeout", "4m") - _, err = tc.Run(cleanupPkgManCmd) - Expect(err).NotTo(HaveOccurred()) - }) - }) -}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go deleted file mode 100644 index c519e4f485..0000000000 --- a/test/e2e/e2e_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Modified from https://github.com/kubernetes-sigs/kubebuilder/tree/39224f0/test/e2e/v3 - -package e2e_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestE2E(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Operator SDK e2e suite") -} diff --git a/test/internal/packagemanifests.go b/test/internal/packagemanifests.go new file mode 100644 index 0000000000..b02288fa4d --- /dev/null +++ b/test/internal/packagemanifests.go @@ -0,0 +1,80 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Modified from https://github.com/kubernetes-sigs/kubebuilder/tree/39224f0/test/e2e/v3 + +package internal + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strings" + + "sigs.k8s.io/kubebuilder/pkg/model/config" +) + +var makefilePackagemanifestsFragment = ` +# Options for "packagemanifests". +ifneq ($(origin FROM_VERSION), undefined) +PKG_FROM_VERSION := --from-version=$(FROM_VERSION) +endif +ifneq ($(origin CHANNEL), undefined) +PKG_CHANNELS := --channel=$(CHANNEL) +endif +ifeq ($(IS_CHANNEL_DEFAULT), 1) +PKG_IS_DEFAULT_CHANNEL := --default-channel +endif +PKG_MAN_OPTS ?= $(FROM_VERSION) $(PKG_CHANNELS) $(PKG_IS_DEFAULT_CHANNEL) + +# Generate package manifests. +packagemanifests: kustomize %s + operator-sdk generate kustomize manifests -q --interactive=false + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate packagemanifests -q --version $(VERSION) $(PKG_MAN_OPTS) +` + +// AddPackagemanifestsTarget will append the packagemanifests target to the makefile +// in order to test the steps described in the docs. +// More info: https://v1-0-x.sdk.operatorframework.io/docs/olm-integration/generation/#package-manifests-formats +func (tc TestContext) AddPackagemanifestsTarget() error { + makefileBytes, err := ioutil.ReadFile(filepath.Join(tc.Dir, "Makefile")) + if err != nil { + return err + } + + b, err := ioutil.ReadFile(filepath.Join(tc.Dir, "PROJECT")) + if err != nil { + return err + } + c := &config.Config{} + if err = c.Unmarshal(b); err != nil { + return err + } + + // add the manifests target when is a Go project. + replaceTarget := "" + if strings.HasPrefix(c.Layout, "go") { + replaceTarget = "manifests" + } + makefilePackagemanifestsFragment = fmt.Sprintf(makefilePackagemanifestsFragment, replaceTarget) + + // update makefile by adding the packagemanifests target + makefileBytes = append([]byte(makefilePackagemanifestsFragment), makefileBytes...) + err = ioutil.WriteFile(filepath.Join(tc.Dir, "Makefile"), makefileBytes, 0644) + if err != nil { + return err + } + return nil +} diff --git a/test/internal/utils.go b/test/internal/utils.go index b1d10bcbb1..3098958e08 100644 --- a/test/internal/utils.go +++ b/test/internal/utils.go @@ -19,7 +19,6 @@ import ( "io/ioutil" "os" "os/exec" - "path/filepath" "regexp" "strings" @@ -28,24 +27,6 @@ import ( kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils" ) -// Makefile fragments to add to the base Makefile just to ensure the packagemanifests feature -const makefilePackagemanifests = ` -# Options for "packagemanifests". -ifneq ($(origin CHANNEL), undefined) -PKG_CHANNELS := --channel=$(CHANNEL) -endif -ifeq ($(IS_CHANNEL_DEFAULT), 1) -PKG_IS_DEFAULT_CHANNEL := --default-channel -endif -PKG_MAN_OPTS ?= $(PKG_CHANNELS) $(PKG_IS_DEFAULT_CHANNEL) - -# Generate package manifests. -packagemanifests: kustomize - operator-sdk generate kustomize manifests -q --interactive=false - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - $(KUSTOMIZE) build config/manifests | operator-sdk generate packagemanifests -q --version $(VERSION) $(PKG_MAN_OPTS) -` - // TestContext wraps kubebuilder's e2e TestContext. type TestContext struct { *kbtestutils.TestContext @@ -77,23 +58,6 @@ func (tc TestContext) KustomizeBuild(dir string) ([]byte, error) { return tc.Run(exec.Command("kustomize", "build", dir)) } -// AddPackagemanifestsTarget will append the packagemanifests target to the makefile -// in order to test the steps described in the docs. -// More info: https://master.sdk.operatorframework.io/docs/olm-integration/generation/#package-manifests-formats -func (tc TestContext) AddPackagemanifestsTarget() error { - makefileBytes, err := ioutil.ReadFile(filepath.Join(tc.Dir, "Makefile")) - if err != nil { - return err - } - - makefileBytes = append([]byte(makefilePackagemanifests), makefileBytes...) - err = ioutil.WriteFile(filepath.Join(tc.Dir, "Makefile"), makefileBytes, 0644) - if err != nil { - return err - } - return nil -} - // ReplaceInFile replaces all instances of old with new in the file at path. func ReplaceInFile(path, old, new string) { info, err := os.Stat(path)