From a613d4a35460542d18ee09408a2e08aefe1cee96 Mon Sep 17 00:00:00 2001 From: Camila Macedo Date: Sat, 19 Sep 2020 11:28:53 +0100 Subject: [PATCH] cleanup: scorecard with its own test --- test/e2e-ansible/e2e_ansible_olm_test.go | 60 +------------ .../e2e-ansible/e2e_ansible_scorecard_test.go | 77 +++++++++++++++++ test/e2e-ansible/e2e_ansible_suite_test.go | 9 ++ test/e2e-go/e2e_go_olm_test.go | 56 ------------ test/e2e-go/e2e_go_scorecard_test.go | 85 +++++++++++++++++++ test/e2e-go/e2e_go_suite_test.go | 4 + test/e2e-helm/e2e_helm_olm_test.go | 65 +------------- test/e2e-helm/e2e_helm_scorecard_test.go | 78 +++++++++++++++++ test/e2e-helm/e2e_helm_suite_test.go | 9 ++ 9 files changed, 265 insertions(+), 178 deletions(-) create mode 100644 test/e2e-ansible/e2e_ansible_scorecard_test.go create mode 100644 test/e2e-go/e2e_go_scorecard_test.go create mode 100644 test/e2e-helm/e2e_helm_scorecard_test.go diff --git a/test/e2e-ansible/e2e_ansible_olm_test.go b/test/e2e-ansible/e2e_ansible_olm_test.go index 091d923beb..1dc85700d3 100644 --- a/test/e2e-ansible/e2e_ansible_olm_test.go +++ b/test/e2e-ansible/e2e_ansible_olm_test.go @@ -15,28 +15,16 @@ package e2e_ansible_test import ( - "encoding/json" - "fmt" "os/exec" - "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" ) var _ = Describe("Integrating ansible Projects with OLM", func() { Context("with operator-sdk", func() { const operatorVersion = "0.0.1" - const ( - OLMBundleValidationTest = "olm-bundle-validation" - OLMCRDsHaveValidationTest = "olm-crds-have-validation" - OLMCRDsHaveResourcesTest = "olm-crds-have-resources" - OLMSpecDescriptorsTest = "olm-spec-descriptors" - OLMStatusDescriptorsTest = "olm-status-descriptors" - ) - It("should generate and run a valid OLM bundle and packagemanifests", func() { By("turning off interactive prompts for all generation tasks.") err := tc.DisableOLMBundleInteractiveMode() @@ -47,18 +35,12 @@ var _ = Describe("Integrating ansible Projects with OLM", func() { Expect(err).NotTo(HaveOccurred()) By("building the operator bundle image") - // Use the existing image tag but with a "-bundle" suffix. - imageSplit := strings.SplitN(tc.ImageName, ":", 2) - bundleImage := imageSplit[0] + "-bundle" - if len(imageSplit) == 2 { - bundleImage += ":" + imageSplit[1] - } - err = tc.Make("bundle-build", "BUNDLE_IMG="+bundleImage) + err = tc.Make("bundle-build", "BUNDLE_IMG="+tc.BundleImageName) Expect(err).NotTo(HaveOccurred()) if isRunningOnKind() { By("loading the bundle image into Kind cluster") - err = tc.LoadImageToKindClusterWithName(bundleImage) + err = tc.LoadImageToKindClusterWithName(tc.BundleImageName) Expect(err).NotTo(HaveOccurred()) } @@ -70,19 +52,6 @@ var _ = Describe("Integrating ansible Projects with OLM", func() { err = tc.Make("packagemanifests", "IMG="+tc.ImageName) Expect(err).NotTo(HaveOccurred()) - By("running basic scorecard tests") - var scorecardOutput v1alpha3.TestList - runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=basic", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err := tc.Run(runScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - Expect(scorecardOutput.Items).To(HaveLen(1)) - Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) - By("running the package") runPkgManCmd := exec.Command(tc.BinaryName, "run", "packagemanifests", "--install-mode", "AllNamespaces", @@ -91,31 +60,6 @@ var _ = Describe("Integrating ansible Projects with OLM", func() { _, err = tc.Run(runPkgManCmd) Expect(err).NotTo(HaveOccurred()) - By("running olm scorecard tests") - runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=olm", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) - Expect(err).To(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - - expected := make(map[string]v1alpha3.State) - expected[OLMBundleValidationTest] = v1alpha3.PassState - expected[OLMCRDsHaveResourcesTest] = v1alpha3.FailState - expected[OLMCRDsHaveValidationTest] = v1alpha3.FailState - expected[OLMSpecDescriptorsTest] = v1alpha3.FailState - expected[OLMStatusDescriptorsTest] = v1alpha3.FailState - - Expect(len(scorecardOutput.Items)).To(Equal(len(expected))) - for a := 0; a < len(scorecardOutput.Items); a++ { - fmt.Println(" - Name: ", scorecardOutput.Items[a].Status.Results[0].Name) - fmt.Println(" Expected: ", expected[scorecardOutput.Items[a].Status.Results[0].Name]) - fmt.Println(" Output: ", scorecardOutput.Items[a].Status.Results[0].State) - Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(expected[scorecardOutput.Items[a].Status.Results[0].Name])) - } - By("destroying the deployed package manifests-formatted operator") cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", tc.ProjectName, "--timeout", "4m") diff --git a/test/e2e-ansible/e2e_ansible_scorecard_test.go b/test/e2e-ansible/e2e_ansible_scorecard_test.go new file mode 100644 index 0000000000..a5bc0f7ae9 --- /dev/null +++ b/test/e2e-ansible/e2e_ansible_scorecard_test.go @@ -0,0 +1,77 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_ansible_test + +import ( + "encoding/json" + "fmt" + "os/exec" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" +) + +var _ = Describe("Testing Ansible Projects with Scorecard", func() { + Context("with operator-sdk", func() { + const ( + OLMBundleValidationTest = "olm-bundle-validation" + OLMCRDsHaveValidationTest = "olm-crds-have-validation" + OLMCRDsHaveResourcesTest = "olm-crds-have-resources" + OLMSpecDescriptorsTest = "olm-spec-descriptors" + OLMStatusDescriptorsTest = "olm-status-descriptors" + ) + + It("should work successfully with scorecard", func() { + By("running basic scorecard tests") + var scorecardOutput v1alpha3.TestList + runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err := tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(scorecardOutput.Items).To(HaveLen(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + + By("running olm scorecard tests") + runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=olm", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) + Expect(err).To(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + + expected := make(map[string]v1alpha3.State) + expected[OLMBundleValidationTest] = v1alpha3.PassState + expected[OLMCRDsHaveResourcesTest] = v1alpha3.FailState + expected[OLMCRDsHaveValidationTest] = v1alpha3.FailState + expected[OLMSpecDescriptorsTest] = v1alpha3.FailState + expected[OLMStatusDescriptorsTest] = v1alpha3.FailState + + Expect(len(scorecardOutput.Items)).To(Equal(len(expected))) + for a := 0; a < len(scorecardOutput.Items); a++ { + fmt.Println(" - Name: ", scorecardOutput.Items[a].Status.Results[0].Name) + fmt.Println(" Expected: ", expected[scorecardOutput.Items[a].Status.Results[0].Name]) + fmt.Println(" Output: ", scorecardOutput.Items[a].Status.Results[0].State) + Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(expected[scorecardOutput.Items[a].Status.Results[0].Name])) + } + }) + }) +}) diff --git a/test/e2e-ansible/e2e_ansible_suite_test.go b/test/e2e-ansible/e2e_ansible_suite_test.go index beeb43cd57..8b9d2b4763 100644 --- a/test/e2e-ansible/e2e_ansible_suite_test.go +++ b/test/e2e-ansible/e2e_ansible_suite_test.go @@ -166,6 +166,11 @@ var _ = BeforeSuite(func(done Done) { "# +kubebuilder:scaffold:rules", rolesForBaseOperator) Expect(err).NotTo(HaveOccurred()) + By("turning off interactive prompts for all generation tasks.") + replace := "operator-sdk generate kustomize manifests" + err = testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false") + Expect(err).NotTo(HaveOccurred()) + By("checking the kustomize setup") err = tc.Make("kustomize") Expect(err).NotTo(HaveOccurred()) @@ -180,6 +185,10 @@ var _ = BeforeSuite(func(done Done) { Expect(err).NotTo(HaveOccurred()) } + By("building the bundle") + err = tc.Make("bundle", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + close(done) }, 360) diff --git a/test/e2e-go/e2e_go_olm_test.go b/test/e2e-go/e2e_go_olm_test.go index 7e84d2a425..bf9919e9b8 100644 --- a/test/e2e-go/e2e_go_olm_test.go +++ b/test/e2e-go/e2e_go_olm_test.go @@ -15,26 +15,16 @@ package e2e_go_test import ( - "encoding/json" "os/exec" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" ) var _ = Describe("Integrating Go Projects with OLM", func() { Context("with operator-sdk", func() { const operatorVersion = "0.0.1" - const ( - OLMBundleValidationTest = "olm-bundle-validation" - OLMCRDsHaveValidationTest = "olm-crds-have-validation" - OLMCRDsHaveResourcesTest = "olm-crds-have-resources" - OLMSpecDescriptorsTest = "olm-spec-descriptors" - OLMStatusDescriptorsTest = "olm-status-descriptors" - ) - It("should generate and run a valid OLM bundle and packagemanifests", func() { By("turning off interactive prompts for all generation tasks.") err := tc.DisableOLMBundleInteractiveMode() @@ -71,52 +61,6 @@ var _ = Describe("Integrating Go Projects with OLM", func() { _, err = tc.Run(runPkgManCmd) Expect(err).NotTo(HaveOccurred()) - By("running basic scorecard tests") - var scorecardOutput v1alpha3.TestList - runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=basic", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err := tc.Run(runScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - Expect(scorecardOutput.Items).To(HaveLen(1)) - Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) - - By("running custom scorecard tests") - runScorecardCmd = exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=custom", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err = tc.Run(runScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - Expect(scorecardOutput.Items).To(HaveLen(2)) - - By("running olm scorecard tests") - runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=olm", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) - Expect(err).To(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - - resultTable := make(map[string]v1alpha3.State) - resultTable[OLMStatusDescriptorsTest] = v1alpha3.FailState - resultTable[OLMCRDsHaveResourcesTest] = v1alpha3.FailState - resultTable[OLMBundleValidationTest] = v1alpha3.PassState - resultTable[OLMSpecDescriptorsTest] = v1alpha3.FailState - resultTable[OLMCRDsHaveValidationTest] = v1alpha3.PassState - - Expect(len(scorecardOutput.Items)).To(Equal(len(resultTable))) - for a := 0; a < len(scorecardOutput.Items); a++ { - Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(resultTable[scorecardOutput.Items[a].Status.Results[0].Name])) - } - By("destroying the deployed package manifests-formatted operator") cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", tc.ProjectName, "--timeout", "4m") diff --git a/test/e2e-go/e2e_go_scorecard_test.go b/test/e2e-go/e2e_go_scorecard_test.go new file mode 100644 index 0000000000..85693ac395 --- /dev/null +++ b/test/e2e-go/e2e_go_scorecard_test.go @@ -0,0 +1,85 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_go_test + +import ( + "encoding/json" + "os/exec" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" +) + +var _ = Describe("Testing Go Projects with Scorecard", func() { + Context("with operator-sdk", func() { + const ( + OLMBundleValidationTest = "olm-bundle-validation" + OLMCRDsHaveValidationTest = "olm-crds-have-validation" + OLMCRDsHaveResourcesTest = "olm-crds-have-resources" + OLMSpecDescriptorsTest = "olm-spec-descriptors" + OLMStatusDescriptorsTest = "olm-status-descriptors" + ) + + It("should work successfully with scorecard", func() { + By("running basic scorecard tests") + var scorecardOutput v1alpha3.TestList + runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err := tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(scorecardOutput.Items).To(HaveLen(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + + By("running custom scorecard tests") + runScorecardCmd = exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=custom", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(scorecardOutput.Items).To(HaveLen(2)) + + By("running olm scorecard tests") + runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=olm", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) + Expect(err).To(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + + resultTable := make(map[string]v1alpha3.State) + resultTable[OLMStatusDescriptorsTest] = v1alpha3.FailState + resultTable[OLMCRDsHaveResourcesTest] = v1alpha3.FailState + resultTable[OLMBundleValidationTest] = v1alpha3.PassState + resultTable[OLMSpecDescriptorsTest] = v1alpha3.FailState + resultTable[OLMCRDsHaveValidationTest] = v1alpha3.PassState + + Expect(len(scorecardOutput.Items)).To(Equal(len(resultTable))) + for a := 0; a < len(scorecardOutput.Items); a++ { + Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(resultTable[scorecardOutput.Items[a].Status.Results[0].Name])) + } + }) + }) +}) diff --git a/test/e2e-go/e2e_go_suite_test.go b/test/e2e-go/e2e_go_suite_test.go index 8b633fb71e..c1cf099ae6 100644 --- a/test/e2e-go/e2e_go_suite_test.go +++ b/test/e2e-go/e2e_go_suite_test.go @@ -151,6 +151,10 @@ var _ = BeforeSuite(func(done Done) { Expect(err).NotTo(HaveOccurred()) } + By("generating the operator bundle") + err = tc.Make("bundle", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + close(done) }, 360) diff --git a/test/e2e-helm/e2e_helm_olm_test.go b/test/e2e-helm/e2e_helm_olm_test.go index 09e77318f5..85874cdac6 100644 --- a/test/e2e-helm/e2e_helm_olm_test.go +++ b/test/e2e-helm/e2e_helm_olm_test.go @@ -15,44 +15,19 @@ package e2e_helm_test import ( - "encoding/json" - "fmt" "os/exec" - "path/filepath" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" - - testutils "github.com/operator-framework/operator-sdk/test/utils" ) var _ = Describe("Integrating Helm Projects with OLM", func() { Context("with operator-sdk", func() { const operatorVersion = "0.0.1" - const ( - OLMBundleValidationTest = "olm-bundle-validation" - OLMCRDsHaveValidationTest = "olm-crds-have-validation" - OLMCRDsHaveResourcesTest = "olm-crds-have-resources" - OLMSpecDescriptorsTest = "olm-spec-descriptors" - OLMStatusDescriptorsTest = "olm-status-descriptors" - ) - - BeforeEach(func() { - By("turning off interactive prompts for all generation tasks.") - replace := "operator-sdk generate kustomize manifests" - err := testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false") - Expect(err).NotTo(HaveOccurred()) - }) - It("should generate and run a valid OLM bundle and packagemanifests", func() { - By("building the bundle") - err := tc.Make("bundle", "IMG="+tc.ImageName) - Expect(err).NotTo(HaveOccurred()) - By("building the operator bundle image") - err = tc.Make("bundle-build", "BUNDLE_IMG="+tc.BundleImageName) + err := tc.Make("bundle-build", "BUNDLE_IMG="+tc.BundleImageName) Expect(err).NotTo(HaveOccurred()) if isRunningOnKind() { @@ -69,19 +44,6 @@ var _ = Describe("Integrating Helm Projects with OLM", func() { err = tc.Make("packagemanifests", "IMG="+tc.ImageName) Expect(err).NotTo(HaveOccurred()) - By("running basic scorecard tests") - var scorecardOutput v1alpha3.TestList - runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=basic", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err := tc.Run(runScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - Expect(scorecardOutput.Items).To(HaveLen(1)) - Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) - By("running the package") runPkgManCmd := exec.Command(tc.BinaryName, "run", "packagemanifests", "--install-mode", "AllNamespaces", @@ -95,31 +57,6 @@ var _ = Describe("Integrating Helm Projects with OLM", func() { "--timeout", "4m") _, err = tc.Run(cleanupPkgManCmd) Expect(err).NotTo(HaveOccurred()) - - By("running olm scorecard tests") - runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=olm", - "--output=json", - "--wait-time=40s") - scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) - Expect(err).To(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) - - expected := make(map[string]v1alpha3.State) - expected[OLMBundleValidationTest] = v1alpha3.PassState - expected[OLMCRDsHaveResourcesTest] = v1alpha3.FailState - expected[OLMCRDsHaveValidationTest] = v1alpha3.FailState - expected[OLMSpecDescriptorsTest] = v1alpha3.FailState - expected[OLMStatusDescriptorsTest] = v1alpha3.FailState - - Expect(len(scorecardOutput.Items)).To(Equal(len(expected))) - for a := 0; a < len(scorecardOutput.Items); a++ { - fmt.Println(" - Name: ", scorecardOutput.Items[a].Status.Results[0].Name) - fmt.Println(" Expected: ", expected[scorecardOutput.Items[a].Status.Results[0].Name]) - fmt.Println(" Output: ", scorecardOutput.Items[a].Status.Results[0].State) - Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(expected[scorecardOutput.Items[a].Status.Results[0].Name])) - } }) }) }) diff --git a/test/e2e-helm/e2e_helm_scorecard_test.go b/test/e2e-helm/e2e_helm_scorecard_test.go new file mode 100644 index 0000000000..9b603d178f --- /dev/null +++ b/test/e2e-helm/e2e_helm_scorecard_test.go @@ -0,0 +1,78 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_helm_test + +import ( + "encoding/json" + "fmt" + "os/exec" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" +) + +var _ = Describe("Testing Helm Projects with Scorecard", func() { + Context("with operator-sdk", func() { + const ( + OLMBundleValidationTest = "olm-bundle-validation" + OLMCRDsHaveValidationTest = "olm-crds-have-validation" + OLMCRDsHaveResourcesTest = "olm-crds-have-resources" + OLMSpecDescriptorsTest = "olm-spec-descriptors" + OLMStatusDescriptorsTest = "olm-status-descriptors" + ) + + It("should work successfully with scorecard", func() { + By("running basic scorecard tests") + var scorecardOutput v1alpha3.TestList + runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err := tc.Run(runScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(scorecardOutput.Items).To(HaveLen(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + + By("running olm scorecard tests") + runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=olm", + "--output=json", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) + Expect(err).To(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + + expected := make(map[string]v1alpha3.State) + expected[OLMBundleValidationTest] = v1alpha3.PassState + expected[OLMCRDsHaveResourcesTest] = v1alpha3.FailState + expected[OLMCRDsHaveValidationTest] = v1alpha3.FailState + expected[OLMSpecDescriptorsTest] = v1alpha3.FailState + expected[OLMStatusDescriptorsTest] = v1alpha3.FailState + + Expect(len(scorecardOutput.Items)).To(Equal(len(expected))) + for a := 0; a < len(scorecardOutput.Items); a++ { + fmt.Println(" - Name: ", scorecardOutput.Items[a].Status.Results[0].Name) + fmt.Println(" Expected: ", expected[scorecardOutput.Items[a].Status.Results[0].Name]) + fmt.Println(" Output: ", scorecardOutput.Items[a].Status.Results[0].State) + Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(expected[scorecardOutput.Items[a].Status.Results[0].Name])) + } + }) + }) +}) diff --git a/test/e2e-helm/e2e_helm_suite_test.go b/test/e2e-helm/e2e_helm_suite_test.go index 3e9563c164..2684f33309 100644 --- a/test/e2e-helm/e2e_helm_suite_test.go +++ b/test/e2e-helm/e2e_helm_suite_test.go @@ -110,6 +110,11 @@ var _ = BeforeSuite(func(done Done) { err = testutils.ReplaceRegexInFile(filepath.Join(tc.Dir, "Dockerfile"), "quay.io/operator-framework/helm-operator:.*", "quay.io/operator-framework/helm-operator:dev") Expect(err).Should(Succeed()) + By("turning off interactive prompts for all generation tasks.") + replace := "operator-sdk generate kustomize manifests" + err = testutils.ReplaceInFile(filepath.Join(tc.Dir, "Makefile"), replace, replace+" --interactive=false") + Expect(err).Should(Succeed()) + By("checking the kustomize setup") err = tc.Make("kustomize") Expect(err).NotTo(HaveOccurred()) @@ -124,6 +129,10 @@ var _ = BeforeSuite(func(done Done) { Expect(err).NotTo(HaveOccurred()) } + By("generating the operator bundle") + err = tc.Make("bundle", "IMG="+tc.ImageName) + Expect(err).NotTo(HaveOccurred()) + close(done) }, 360)