From 343857a2444141ae659a06cbe4057b3621b38db7 Mon Sep 17 00:00:00 2001 From: Bruno Dias Date: Tue, 21 Jan 2020 16:49:40 -0300 Subject: [PATCH] ISV Operator Automation --- cmd/openshift-tests/e2e.go | 13 +- test/extended/include.go | 1 + test/extended/isv/isv.go | 206 ++++++++++++++++++ test/extended/testdata/bindata.go | 59 +++++ .../extended/testdata/isv/operator_group.yaml | 7 + test/extended/testdata/isv/subscription.yaml | 12 + test/extended/util/test.go | 5 +- 7 files changed, 301 insertions(+), 2 deletions(-) create mode 100644 test/extended/isv/isv.go create mode 100644 test/extended/testdata/isv/operator_group.yaml create mode 100644 test/extended/testdata/isv/subscription.yaml diff --git a/cmd/openshift-tests/e2e.go b/cmd/openshift-tests/e2e.go index 76855dc4e311..13d06321596d 100644 --- a/cmd/openshift-tests/e2e.go +++ b/cmd/openshift-tests/e2e.go @@ -29,7 +29,7 @@ var staticSuites = []*ginkgo.TestSuite{ Only the portion of the openshift/conformance test suite that run in parallel. `), Matches: func(name string) bool { - return strings.Contains(name, "[Suite:openshift/conformance/parallel") + return strings.Contains(name, "[Suite:openshift/conformance/parallel") && !strings.Contains(name, "[Suite:openshift/isv") }, Parallelism: 30, MaximumAllowedFlakes: 15, @@ -155,6 +155,17 @@ var staticSuites = []*ginkgo.TestSuite{ Count: 15, TestTimeout: 20 * time.Minute, }, + { + Name: "openshift/isv", + Description: templates.LongDesc(` + This test suite verifies the Operators execution on Openshift + `), + Matches: func(name string) bool { + return strings.Contains(name, "[Suite:openshift/isv]") + }, + Parallelism: 5, + TestTimeout: 60 * time.Minute, + }, { Name: "all", Description: templates.LongDesc(` diff --git a/test/extended/include.go b/test/extended/include.go index bd2bb53b999f..7606c826d13e 100644 --- a/test/extended/include.go +++ b/test/extended/include.go @@ -53,4 +53,5 @@ import ( _ "github.com/openshift/origin/test/extended/security" _ "github.com/openshift/origin/test/extended/templates" _ "github.com/openshift/origin/test/extended/user" + _ "github.com/openshift/origin/test/extended/isv" ) diff --git a/test/extended/isv/isv.go b/test/extended/isv/isv.go new file mode 100644 index 000000000000..ee66f1cd9b3e --- /dev/null +++ b/test/extended/isv/isv.go @@ -0,0 +1,206 @@ +package isv + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/storage/names" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type packagemanifest struct { + name string + supportsOwnNamespace bool + supportsSingleNamespace bool + supportsAllNamespaces bool + csvVersion string + namespace string + defaultChannel string + catalogSource string + catalogSourceNamespace string +} + +var _ = g.Describe("[Suite:openshift/isv] Operator", func() { + var ( + oc = exutil.NewCLI("isv", exutil.KubeConfigPath()) + catalogLabels = []string{"redhat-operators", "certified-operators"} + output, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", "-l catalog="+catalogLabels[0], "-o=jsonpath={range .items[*].metadata}{.name}{'\\n'}{end}").Output() + redhatPackages = strings.Split(output, "\n") + output1, _ = oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", "-l catalog="+catalogLabels[1], "-o=jsonpath={range .items[*].metadata}{.name}{'\\n'}{end}").Output() + certifiedPackages = strings.Split(output1, "\n") + packages = append(redhatPackages, certifiedPackages...) + //packages = []string{"serverless-operator"} + currentPackage packagemanifest + ) + defer g.GinkgoRecover() + + g.AfterEach(func() { + if currentPackage.supportsSingleNamespace || currentPackage.supportsOwnNamespace { + _, err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("ns", currentPackage.namespace).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } + }) + + for i := range packages { + + isv := packages[i] + g.It(isv+" should work properly", func() { + g.By("by installing", func() { + currentPackage = createSubscription(isv, oc) + checkDeployment(currentPackage, oc) + }) + g.By("by unistalling", func() { + removeOperatorDependencies(currentPackage, oc, true) + }) + }) + } + +}) + +func checkOperatorInstallModes(p packagemanifest, oc *exutil.CLI) packagemanifest { + supportsAllNamespaces, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", p.name, "-o=jsonpath={.status.channels[?(.name=='"+p.defaultChannel+"')].currentCSVDesc.installModes[?(.type=='AllNamespaces')].supported}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + supportsAllNamespacesAsBool, _ := strconv.ParseBool(supportsAllNamespaces) + + supportsSingleNamespace, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", p.name, "-o=jsonpath={.status.channels[?(.name=='"+p.defaultChannel+"')].currentCSVDesc.installModes[?(.type=='SingleNamespace')].supported}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + supportsSingleNamespaceAsBool, _ := strconv.ParseBool(supportsSingleNamespace) + + supportsOwnNamespace, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", p.name, "-o=jsonpath={.status.channels[?(.name=='"+p.defaultChannel+"')].currentCSVDesc.installModes[?(.type=='OwnNamespace')].supported}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + supportsOwnNamespaceAsBool, _ := strconv.ParseBool(supportsOwnNamespace) + + p.supportsAllNamespaces = supportsAllNamespacesAsBool + p.supportsSingleNamespace = supportsSingleNamespaceAsBool + p.supportsOwnNamespace = supportsOwnNamespaceAsBool + return p +} + +func createSubscription(isv string, oc *exutil.CLI) packagemanifest { + + msg, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", isv, "-o=jsonpath={.status.catalogSource}:{.status.catalogSourceNamespace}:{.status.defaultChannel}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + packageData := strings.Split(msg, ":") + p := packagemanifest{catalogSource: packageData[0], catalogSourceNamespace: packageData[1], defaultChannel: packageData[2], name: isv} + + csvVersion, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("packagemanifest", p.name, "-o=jsonpath={.status.channels[?(.name=='"+p.defaultChannel+"')].currentCSV}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + p.csvVersion = csvVersion + + p = checkOperatorInstallModes(p, oc) + + if p.supportsSingleNamespace || p.supportsOwnNamespace { + p = createNamespace(p, oc) + createOperatorGroup(p, oc) + } else if p.supportsAllNamespaces { + p.namespace = "openshift-operators" + } else { + g.Skip("Install Modes AllNamespaces and SingleNamespace are disabled for Operator: " + isv) + } + + templateSubscriptionYAML := writeSubscription(p) + _, err = oc.SetNamespace(p.namespace).AsAdmin().Run("create").Args("-f", templateSubscriptionYAML).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return p +} + +func createNamespace(p packagemanifest, oc *exutil.CLI) packagemanifest { + p.namespace = names.SimpleNameGenerator.GenerateName("test-operators-") + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("ns", p.namespace).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return p +} + +func createOperatorGroup(p packagemanifest, oc *exutil.CLI) { + + templateOperatorGroupYAML := writeOperatorGroup(p.namespace) + _, err := oc.SetNamespace(p.namespace).AsAdmin().Run("create").Args("-f", templateOperatorGroupYAML).Output() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func writeOperatorGroup(namespace string) (templateOperatorYAML string) { + isvBaseDir := exutil.FixturePath("testdata", "isv") + operatorGroupYAML := filepath.Join(isvBaseDir, "operator_group.yaml") + fileOperatorGroup, _ := os.Open(operatorGroupYAML) + operatorGroup, _ := ioutil.ReadAll(fileOperatorGroup) + operatorGroupTemplate := string(operatorGroup) + templateOperatorYAML = strings.ReplaceAll(operatorGroupYAML, "operator_group.yaml", "operator_group_"+namespace+"_.yaml") + operatorGroupString := strings.ReplaceAll(operatorGroupTemplate, "$OPERATOR_NAMESPACE", namespace) + ioutil.WriteFile(templateOperatorYAML, []byte(operatorGroupString), 0644) + return +} + +func writeSubscription(p packagemanifest) (templateSubscriptionYAML string) { + isvBaseDir := exutil.FixturePath("testdata", "isv") + subscriptionYAML := filepath.Join(isvBaseDir, "subscription.yaml") + fileSubscription, _ := os.Open(subscriptionYAML) + subscription, _ := ioutil.ReadAll(fileSubscription) + subscriptionTemplate := string(subscription) + + templateSubscriptionYAML = strings.ReplaceAll(subscriptionYAML, "subscription.yaml", "subscription_"+p.csvVersion+"_.yaml") + operatorSubscription := strings.ReplaceAll(subscriptionTemplate, "$OPERATOR_PACKAGE_NAME", p.name) + operatorSubscription = strings.ReplaceAll(operatorSubscription, "$OPERATOR_CHANNEL", "\""+p.defaultChannel+"\"") + operatorSubscription = strings.ReplaceAll(operatorSubscription, "$OPERATOR_NAMESPACE", p.namespace) + operatorSubscription = strings.ReplaceAll(operatorSubscription, "$OPERATOR_SOURCE", p.catalogSource) + operatorSubscription = strings.ReplaceAll(operatorSubscription, "$OPERATOR_CATALOG_NAMESPACE", p.catalogSourceNamespace) + operatorSubscription = strings.ReplaceAll(operatorSubscription, "$OPERATOR_CURRENT_CSV_VERSION", p.csvVersion) + ioutil.WriteFile(templateSubscriptionYAML, []byte(operatorSubscription), 0644) + e2e.Logf("Subscription: %s", operatorSubscription) + return +} +func checkDeployment(p packagemanifest, oc *exutil.CLI) { + poolErr := wait.Poll(10*time.Second, 300*time.Second, func() (bool, error) { + msg, _ := oc.SetNamespace(p.namespace).AsAdmin().Run("get").Args("csv", p.csvVersion, "-o=jsonpath={.status.phase}").Output() + if strings.Contains(msg, "Succeeded") { + return true, nil + } + return false, nil + }) + if poolErr != nil { + removeOperatorDependencies(p, oc, false) + g.Fail("Could not obtain CSV:" + p.csvVersion) + } +} + +func removeOperatorDependencies(p packagemanifest, oc *exutil.CLI, checkDeletion bool) { + ip, _ := oc.SetNamespace(p.namespace).AsAdmin().Run("get").Args("sub", p.name, "-o=jsonpath={.status.installplan.name}").Output() + e2e.Logf("IP: %s", ip) + if len(strings.TrimSpace(ip)) > 0 { + msg, _ := oc.SetNamespace(p.namespace).AsAdmin().Run("get").Args("ip", ip, "-o=jsonpath={.spec.clusterServiceVersionNames}").Output() + msg = strings.ReplaceAll(msg, "[", "") + msg = strings.ReplaceAll(msg, "]", "") + e2e.Logf("CSVS: %s", msg) + csvs := strings.Split(msg, " ") + for i := range csvs { + e2e.Logf("CSV_: %s", csvs[i]) + msg, err := oc.SetNamespace(p.namespace).AsAdmin().Run("delete").Args("csv", csvs[i]).Output() + if checkDeletion { + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(msg).To(o.ContainSubstring("deleted")) + } + } + + subsOutput, _ := oc.SetNamespace(p.namespace).AsAdmin().Run("get").Args("subs", "-o=jsonpath={range .items[?(.status.installplan.name=='"+ip+"')].metadata}{.name}{' '}").Output() + e2e.Logf("SUBS OUTPUT: %s", subsOutput) + if len(strings.TrimSpace(subsOutput)) > 0 { + subs := strings.Split(subsOutput, " ") + e2e.Logf("SUBS: %s", subs) + for i := range subs { + e2e.Logf("SUB_: %s", subs[i]) + msg, err := oc.SetNamespace(p.namespace).AsAdmin().Run("delete").Args("subs", subs[i]).Output() + if checkDeletion { + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(msg).To(o.ContainSubstring("deleted")) + } + } + } + } +} diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index 44388956d670..43aeb9141a44 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -384,6 +384,8 @@ // test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json // test/extended/testdata/imagestream-jenkins-slave-pods.yaml // test/extended/testdata/imagestreamtag-jenkins-slave-pods.yaml +// test/extended/testdata/isv/operator_group.yaml +// test/extended/testdata/isv/subscription.yaml // test/extended/testdata/jenkins-plugin/build-job-clone.xml // test/extended/testdata/jenkins-plugin/build-job-slave.xml // test/extended/testdata/jenkins-plugin/build-job.xml @@ -50875,6 +50877,57 @@ func testExtendedTestdataImagestreamtagJenkinsSlavePodsYaml() (*asset, error) { return a, nil } +var _testExtendedTestdataIsvOperator_groupYaml = []byte(`apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: test-operators +spec: + targetNamespaces: + - $OPERATOR_NAMESPACE`) + +func testExtendedTestdataIsvOperator_groupYamlBytes() ([]byte, error) { + return _testExtendedTestdataIsvOperator_groupYaml, nil +} + +func testExtendedTestdataIsvOperator_groupYaml() (*asset, error) { + bytes, err := testExtendedTestdataIsvOperator_groupYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/isv/operator_group.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataIsvSubscriptionYaml = []byte(`apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: $OPERATOR_PACKAGE_NAME + namespace: $OPERATOR_NAMESPACE +spec: + channel: $OPERATOR_CHANNEL + installPlanApproval: Automatic + name: $OPERATOR_PACKAGE_NAME + source: $OPERATOR_SOURCE + sourceNamespace: $OPERATOR_CATALOG_NAMESPACE + startingCSV: $OPERATOR_CURRENT_CSV_VERSION`) + +func testExtendedTestdataIsvSubscriptionYamlBytes() ([]byte, error) { + return _testExtendedTestdataIsvSubscriptionYaml, nil +} + +func testExtendedTestdataIsvSubscriptionYaml() (*asset, error) { + bytes, err := testExtendedTestdataIsvSubscriptionYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/isv/subscription.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _testExtendedTestdataJenkinsPluginBuildJobCloneXml = []byte(` @@ -58638,6 +58691,8 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json": testExtendedTestdataImage_ecosystemPerlHotdeployPerlJson, "test/extended/testdata/imagestream-jenkins-slave-pods.yaml": testExtendedTestdataImagestreamJenkinsSlavePodsYaml, "test/extended/testdata/imagestreamtag-jenkins-slave-pods.yaml": testExtendedTestdataImagestreamtagJenkinsSlavePodsYaml, + "test/extended/testdata/isv/operator_group.yaml": testExtendedTestdataIsvOperator_groupYaml, + "test/extended/testdata/isv/subscription.yaml": testExtendedTestdataIsvSubscriptionYaml, "test/extended/testdata/jenkins-plugin/build-job-clone.xml": testExtendedTestdataJenkinsPluginBuildJobCloneXml, "test/extended/testdata/jenkins-plugin/build-job-slave.xml": testExtendedTestdataJenkinsPluginBuildJobSlaveXml, "test/extended/testdata/jenkins-plugin/build-job.xml": testExtendedTestdataJenkinsPluginBuildJobXml, @@ -59351,6 +59406,10 @@ var _bintree = &bintree{nil, map[string]*bintree{ }}, "imagestream-jenkins-slave-pods.yaml": &bintree{testExtendedTestdataImagestreamJenkinsSlavePodsYaml, map[string]*bintree{}}, "imagestreamtag-jenkins-slave-pods.yaml": &bintree{testExtendedTestdataImagestreamtagJenkinsSlavePodsYaml, map[string]*bintree{}}, + "isv": &bintree{nil, map[string]*bintree{ + "operator_group.yaml": &bintree{testExtendedTestdataIsvOperator_groupYaml, map[string]*bintree{}}, + "subscription.yaml": &bintree{testExtendedTestdataIsvSubscriptionYaml, map[string]*bintree{}}, + }}, "jenkins-plugin": &bintree{nil, map[string]*bintree{ "build-job-clone.xml": &bintree{testExtendedTestdataJenkinsPluginBuildJobCloneXml, map[string]*bintree{}}, "build-job-slave.xml": &bintree{testExtendedTestdataJenkinsPluginBuildJobSlaveXml, map[string]*bintree{}}, diff --git a/test/extended/testdata/isv/operator_group.yaml b/test/extended/testdata/isv/operator_group.yaml new file mode 100644 index 000000000000..421ec9a9f17b --- /dev/null +++ b/test/extended/testdata/isv/operator_group.yaml @@ -0,0 +1,7 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: test-operators +spec: + targetNamespaces: + - $OPERATOR_NAMESPACE \ No newline at end of file diff --git a/test/extended/testdata/isv/subscription.yaml b/test/extended/testdata/isv/subscription.yaml new file mode 100644 index 000000000000..f9188f3f6175 --- /dev/null +++ b/test/extended/testdata/isv/subscription.yaml @@ -0,0 +1,12 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: $OPERATOR_PACKAGE_NAME + namespace: $OPERATOR_NAMESPACE +spec: + channel: $OPERATOR_CHANNEL + installPlanApproval: Automatic + name: $OPERATOR_PACKAGE_NAME + source: $OPERATOR_SOURCE + sourceNamespace: $OPERATOR_CATALOG_NAMESPACE + startingCSV: $OPERATOR_CURRENT_CSV_VERSION \ No newline at end of file diff --git a/test/extended/util/test.go b/test/extended/util/test.go index 4e5635bed477..cd3c5820801d 100644 --- a/test/extended/util/test.go +++ b/test/extended/util/test.go @@ -240,7 +240,7 @@ func (r *ginkgoTestRenamer) maybeRenameTest(name string, node types.TestNode) { } } - if !r.excludedTestsFilter.MatchString(name) { + if !r.excludedTestsFilter.MatchString(name) && !strings.Contains(name, "openshift/isv") { isSerial := strings.Contains(name, "[Serial]") isConformance := strings.Contains(name, "[Conformance]") switch { @@ -555,6 +555,9 @@ var ( "[Suite:openshift/csi]": { `External Storage \[Driver:`, }, + "[Suite:openshift/isv]": { + `\[Suite:openshift/isv\]`, + }, } // labelExcludes temporarily block tests out of a specific suite