From 08fa066092c843b793ef1e8738ff53786ec87d3d Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Fri, 17 Jul 2020 12:53:35 -0700 Subject: [PATCH 01/12] add basic cli call --- test/e2e/e2e_suite.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 36b36819c5..42ff33870f 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -211,6 +211,11 @@ var _ = Describe("operator-sdk", func() { "--timeout", "4m") _, err = tc.Run(cleanupPkgManCmd) Expect(err).NotTo(HaveOccurred()) + + By("running scorecard tests") + scorecardCmd := exec.Command(tc.BinaryName, "scorecard") + _, err = tc.Run(scorecardCmd) + Expect(err).NotTo(HaveOccurred()) }) }) }) From bba255b0d3eab1d274c71f71525a298e3cf8a2db Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Thu, 23 Jul 2020 15:19:53 -0700 Subject: [PATCH 02/12] parse json output --- test/e2e/e2e_suite.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 42ff33870f..221a84e1a6 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -18,6 +18,7 @@ package e2e import ( "bytes" + "encoding/json" "fmt" "io/ioutil" "os" @@ -31,6 +32,7 @@ import ( . "github.com/onsi/gomega" //nolint:golint kbtestutils "sigs.k8s.io/kubebuilder/test/e2e/utils" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" testutils "github.com/operator-framework/operator-sdk/test/internal" ) @@ -213,9 +215,18 @@ var _ = Describe("operator-sdk", func() { Expect(err).NotTo(HaveOccurred()) By("running scorecard tests") - scorecardCmd := exec.Command(tc.BinaryName, "scorecard") - _, err = tc.Run(scorecardCmd) + var scorecardOutput v1alpha3.TestList + runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--skip-cleanup=true", + "--wait-time=40s") + scorecardOutputBytes, err := tc.Run(runScorecardCmd) Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + if err != nil { + fmt.Printf("Error parsing JSON output") + } }) }) }) From c37ffabfeaedcb64f28516cd3ed405fe7365358c Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Thu, 23 Jul 2020 15:29:58 -0700 Subject: [PATCH 03/12] check testlist to have 1 result --- test/e2e/e2e_suite.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 221a84e1a6..ae145f1675 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -224,9 +224,8 @@ var _ = Describe("operator-sdk", func() { scorecardOutputBytes, err := tc.Run(runScorecardCmd) Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - if err != nil { - fmt.Printf("Error parsing JSON output") - } + Expect(err).NotTo(HaveOccurred()) + Expect(len(scorecardOutput.Items)).To(Equal(1)) }) }) }) From a5255dbf2e38cd4fbd7ad3c35d529645eda2acb4 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Thu, 23 Jul 2020 15:36:18 -0700 Subject: [PATCH 04/12] verify test passing --- test/e2e/e2e_suite.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index ae145f1675..06937954aa 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -226,6 +226,7 @@ var _ = Describe("operator-sdk", func() { err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) Expect(err).NotTo(HaveOccurred()) Expect(len(scorecardOutput.Items)).To(Equal(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) }) }) }) From 720592e81814d282b8aa4d562e6dec72e9b0e223 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Thu, 23 Jul 2020 15:37:48 -0700 Subject: [PATCH 05/12] olm scorecard added --- test/e2e/e2e_suite.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 06937954aa..d7a4a415d0 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -214,7 +214,7 @@ var _ = Describe("operator-sdk", func() { _, err = tc.Run(cleanupPkgManCmd) Expect(err).NotTo(HaveOccurred()) - By("running scorecard tests") + By("running basic scorecard tests") var scorecardOutput v1alpha3.TestList runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", "--selector=suite=basic", @@ -227,6 +227,19 @@ var _ = Describe("operator-sdk", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(scorecardOutput.Items)).To(Equal(1)) Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + + By("running olm scorecard tests") + runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", + "--selector=suite=basic", + "--output=json", + "--skip-cleanup=true", + "--wait-time=40s") + scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + Expect(len(scorecardOutput.Items)).To(Equal(1)) + Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) }) }) }) From 64bd778e255834d32d3b060a6546d60833067e89 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Thu, 23 Jul 2020 15:38:08 -0700 Subject: [PATCH 06/12] olm scorecard flag --- test/e2e/e2e_suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index d7a4a415d0..3a9124308d 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -230,7 +230,7 @@ var _ = Describe("operator-sdk", func() { By("running olm scorecard tests") runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", - "--selector=suite=basic", + "--selector=suite=olm", "--output=json", "--skip-cleanup=true", "--wait-time=40s") From a7d1cacd7b23dce11f3a062d3ec95a34468349b7 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Fri, 24 Jul 2020 08:04:56 -0700 Subject: [PATCH 07/12] run tests before cleanup --- test/e2e/e2e_suite.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 3a9124308d..10e2358d76 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -206,14 +206,6 @@ var _ = Describe("operator-sdk", func() { _, err = tc.Run(runPkgManCmd) Expect(err).NotTo(HaveOccurred()) - By("destroying the deployed package manifests-formatted operator") - cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", "packagemanifests", - "--operator-namespace", tc.Kubectl.Namespace, - "--operator-version", operatorVersion, - "--timeout", "4m") - _, err = tc.Run(cleanupPkgManCmd) - Expect(err).NotTo(HaveOccurred()) - By("running basic scorecard tests") var scorecardOutput v1alpha3.TestList runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", @@ -240,6 +232,14 @@ var _ = Describe("operator-sdk", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(scorecardOutput.Items)).To(Equal(1)) Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + + By("destroying the deployed package manifests-formatted operator") + cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", "packagemanifests", + "--operator-namespace", tc.Kubectl.Namespace, + "--operator-version", operatorVersion, + "--timeout", "4m") + _, err = tc.Run(cleanupPkgManCmd) + Expect(err).NotTo(HaveOccurred()) }) }) }) From 1d212627f4cc164b95c96cf383f71f2c3ca5d0d0 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Fri, 24 Jul 2020 13:43:10 -0700 Subject: [PATCH 08/12] fix olm test --- test/e2e/e2e_suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 10e2358d76..fd431d4544 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -229,7 +229,7 @@ var _ = Describe("operator-sdk", func() { scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) Expect(err).NotTo(HaveOccurred()) err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) - Expect(err).NotTo(HaveOccurred()) + Expect(err).To(HaveOccurred()) Expect(len(scorecardOutput.Items)).To(Equal(1)) Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) From f9f1aed8ccccd8de7dc99b3e454b40d84932d521 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Fri, 24 Jul 2020 14:03:54 -0700 Subject: [PATCH 09/12] olm expected result table --- test/e2e/e2e_suite.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index fd431d4544..69545063b4 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -227,11 +227,20 @@ var _ = Describe("operator-sdk", func() { "--skip-cleanup=true", "--wait-time=40s") scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) - Expect(err).NotTo(HaveOccurred()) - err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) Expect(err).To(HaveOccurred()) - Expect(len(scorecardOutput.Items)).To(Equal(1)) - Expect(scorecardOutput.Items[0].Status.Results[0].State).To(Equal(v1alpha3.PassState)) + err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) + Expect(err).NotTo(HaveOccurred()) + + var resultTable map[string]v1alpha3.State + resultTable["olm-status-descriptors"] = v1alpha3.FailState + resultTable["olm-crds-have-resources"] = v1alpha3.FailState + resultTable["olm-bundle-validation"] = v1alpha3.PassState + resultTable["olm-spec-descriptors"] = v1alpha3.FailState + resultTable["olm-crds-have-validation"] = v1alpha3.PassState + + for a := 0; a < len(scorecardOutput.Items); a++ { + Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(resultTable[scorecardOutput.Items[a].Status.Results[0].Name])) + } By("destroying the deployed package manifests-formatted operator") cleanupPkgManCmd := exec.Command(tc.BinaryName, "cleanup", "packagemanifests", From a29956e81360652e043436a1be78ba97d39a4c26 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Fri, 24 Jul 2020 14:37:12 -0700 Subject: [PATCH 10/12] init map --- test/e2e/e2e_suite.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 69545063b4..8740177f31 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -232,6 +232,7 @@ var _ = Describe("operator-sdk", func() { Expect(err).NotTo(HaveOccurred()) var resultTable map[string]v1alpha3.State + resultTable = make(map[string]v1alpha3.State) resultTable["olm-status-descriptors"] = v1alpha3.FailState resultTable["olm-crds-have-resources"] = v1alpha3.FailState resultTable["olm-bundle-validation"] = v1alpha3.PassState From 30350ce0280894976c4dbd286ef037aedbc1fde4 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Wed, 29 Jul 2020 08:06:10 -0700 Subject: [PATCH 11/12] linter fix --- test/e2e/e2e_suite.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 8740177f31..9c46f1a955 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -231,8 +231,7 @@ var _ = Describe("operator-sdk", func() { err = json.Unmarshal(scorecardOutputBytes, &scorecardOutput) Expect(err).NotTo(HaveOccurred()) - var resultTable map[string]v1alpha3.State - resultTable = make(map[string]v1alpha3.State) + resultTable := make(map[string]v1alpha3.State) resultTable["olm-status-descriptors"] = v1alpha3.FailState resultTable["olm-crds-have-resources"] = v1alpha3.FailState resultTable["olm-bundle-validation"] = v1alpha3.PassState From 940dc269c043a8cdd09ec753a155c81c65512831 Mon Sep 17 00:00:00 2001 From: Ish Shah Date: Fri, 31 Jul 2020 08:22:46 -0700 Subject: [PATCH 12/12] rm skip cleanup, add const test names --- test/e2e/e2e_suite.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/e2e/e2e_suite.go b/test/e2e/e2e_suite.go index 9c46f1a955..8967d285a3 100644 --- a/test/e2e/e2e_suite.go +++ b/test/e2e/e2e_suite.go @@ -36,6 +36,14 @@ import ( testutils "github.com/operator-framework/operator-sdk/test/internal" ) +const ( + OLMBundleValidationTest = "olm-bundle-validation" + OLMCRDsHaveValidationTest = "olm-crds-have-validation" + OLMCRDsHaveResourcesTest = "olm-crds-have-resources" + OLMSpecDescriptorsTest = "olm-spec-descriptors" + OLMStatusDescriptorsTest = "olm-status-descriptors" +) + var _ = Describe("operator-sdk", func() { Context("with the new project layout", func() { var ( @@ -211,7 +219,6 @@ var _ = Describe("operator-sdk", func() { runScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", "--selector=suite=basic", "--output=json", - "--skip-cleanup=true", "--wait-time=40s") scorecardOutputBytes, err := tc.Run(runScorecardCmd) Expect(err).NotTo(HaveOccurred()) @@ -224,7 +231,6 @@ var _ = Describe("operator-sdk", func() { runOLMScorecardCmd := exec.Command(tc.BinaryName, "scorecard", "bundle", "--selector=suite=olm", "--output=json", - "--skip-cleanup=true", "--wait-time=40s") scorecardOutputBytes, err = tc.Run(runOLMScorecardCmd) Expect(err).To(HaveOccurred()) @@ -232,12 +238,13 @@ var _ = Describe("operator-sdk", func() { Expect(err).NotTo(HaveOccurred()) resultTable := make(map[string]v1alpha3.State) - resultTable["olm-status-descriptors"] = v1alpha3.FailState - resultTable["olm-crds-have-resources"] = v1alpha3.FailState - resultTable["olm-bundle-validation"] = v1alpha3.PassState - resultTable["olm-spec-descriptors"] = v1alpha3.FailState - resultTable["olm-crds-have-validation"] = v1alpha3.PassState + resultTable[OLMStatusDescriptorsTest] = v1alpha3.FailState + resultTable[OLMCRDsHaveResourcesTest] = v1alpha3.FailState + resultTable[OLMBundleValidationTest] = v1alpha3.PassState + resultTable[OLMSpecDescriptorsTest] = v1alpha3.FailState + resultTable[OLMCRDsHaveValidationTest] = v1alpha3.PassState + Expect(len(scorecardOutput.Items)).To(Equal(len(resultTable))) for a := 0; a < len(scorecardOutput.Items); a++ { Expect(scorecardOutput.Items[a].Status.Results[0].State).To(Equal(resultTable[scorecardOutput.Items[a].Status.Results[0].Name])) }