diff --git a/cmd/operator-sdk/scorecard/cmd.go b/cmd/operator-sdk/scorecard/cmd.go index 7acc21d9b0a..735ff0237b8 100644 --- a/cmd/operator-sdk/scorecard/cmd.go +++ b/cmd/operator-sdk/scorecard/cmd.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/operator-framework/operator-sdk/internal/pkg/scaffold" + "github.com/operator-framework/operator-sdk/internal/pkg/scorecard" "github.com/operator-framework/operator-sdk/version" log "github.com/sirupsen/logrus" @@ -26,52 +27,31 @@ import ( "github.com/spf13/viper" ) -// scorecardConfig stores all scorecard config passed as flags -type scorecardConfig struct { - namespace string - kubeconfigPath string - initTimeout int - olmDeployed bool - csvPath string - basicTests bool - olmTests bool - tenantTests bool - namespacedManifest string - globalManifest string - crManifest string - proxyImage string - proxyPullPolicy string - crdsDir string - verbose bool -} - -var scConf scorecardConfig - func NewCmd() *cobra.Command { scorecardCmd := &cobra.Command{ Use: "scorecard", Short: "Run scorecard tests", Long: `Runs blackbox scorecard tests on an operator `, - RunE: ScorecardTests, + RunE: scorecard.ScorecardTests, } - scorecardCmd.Flags().StringVar(&ScorecardConf, ConfigOpt, "", "config file (default is /.osdk-yaml)") - scorecardCmd.Flags().StringVar(&scConf.namespace, NamespaceOpt, "", "Namespace of custom resource created in cluster") - scorecardCmd.Flags().StringVar(&scConf.kubeconfigPath, KubeconfigOpt, "", "Path to kubeconfig of custom resource created in cluster") - scorecardCmd.Flags().IntVar(&scConf.initTimeout, InitTimeoutOpt, 10, "Timeout for status block on CR to be created in seconds") - scorecardCmd.Flags().BoolVar(&scConf.olmDeployed, OlmDeployedOpt, false, "The OLM has deployed the operator. Use only the CSV for test data") - scorecardCmd.Flags().StringVar(&scConf.csvPath, CSVPathOpt, "", "Path to CSV being tested") - scorecardCmd.Flags().BoolVar(&scConf.basicTests, BasicTestsOpt, true, "Enable basic operator checks") - scorecardCmd.Flags().BoolVar(&scConf.olmTests, OLMTestsOpt, true, "Enable OLM integration checks") - scorecardCmd.Flags().BoolVar(&scConf.tenantTests, TenantTestsOpt, false, "Enable good tenant checks") - scorecardCmd.Flags().StringVar(&scConf.namespacedManifest, NamespacedManifestOpt, "", "Path to manifest for namespaced resources (e.g. RBAC and Operator manifest)") - scorecardCmd.Flags().StringVar(&scConf.globalManifest, GlobalManifestOpt, "", "Path to manifest for Global resources (e.g. CRD manifests)") - scorecardCmd.Flags().StringVar(&scConf.crManifest, CRManifestOpt, "", "Path to manifest for Custom Resource (required)") - scorecardCmd.Flags().StringVar(&scConf.proxyImage, ProxyImageOpt, fmt.Sprintf("quay.io/operator-framework/scorecard-proxy:%s", strings.TrimSuffix(version.Version, "+git")), "Image name for scorecard proxy") - scorecardCmd.Flags().StringVar(&scConf.proxyPullPolicy, ProxyPullPolicyOpt, "Always", "Pull policy for scorecard proxy image") - scorecardCmd.Flags().StringVar(&scConf.crdsDir, "crds-dir", scaffold.CRDsDir, "Directory containing CRDs (all CRD manifest filenames must have the suffix 'crd.yaml')") - scorecardCmd.Flags().BoolVar(&scConf.verbose, VerboseOpt, false, "Enable verbose logging") + scorecardCmd.Flags().String(scorecard.ConfigOpt, "", "config file (default is /.osdk-yaml)") + scorecardCmd.Flags().String(scorecard.NamespaceOpt, "", "Namespace of custom resource created in cluster") + scorecardCmd.Flags().String(scorecard.KubeconfigOpt, "", "Path to kubeconfig of custom resource created in cluster") + scorecardCmd.Flags().Int(scorecard.InitTimeoutOpt, 10, "Timeout for status block on CR to be created in seconds") + scorecardCmd.Flags().Bool(scorecard.OlmDeployedOpt, false, "The OLM has deployed the operator. Use only the CSV for test data") + scorecardCmd.Flags().String(scorecard.CSVPathOpt, "", "Path to CSV being tested") + scorecardCmd.Flags().Bool(scorecard.BasicTestsOpt, true, "Enable basic operator checks") + scorecardCmd.Flags().Bool(scorecard.OLMTestsOpt, true, "Enable OLM integration checks") + scorecardCmd.Flags().Bool(scorecard.TenantTestsOpt, false, "Enable good tenant checks") + scorecardCmd.Flags().String(scorecard.NamespacedManifestOpt, "", "Path to manifest for namespaced resources (e.g. RBAC and Operator manifest)") + scorecardCmd.Flags().String(scorecard.GlobalManifestOpt, "", "Path to manifest for Global resources (e.g. CRD manifests)") + scorecardCmd.Flags().String(scorecard.CRManifestOpt, "", "Path to manifest for Custom Resource (required)") + scorecardCmd.Flags().String(scorecard.ProxyImageOpt, fmt.Sprintf("quay.io/operator-framework/scorecard-proxy:%s", strings.TrimSuffix(version.Version, "+git")), "Image name for scorecard proxy") + scorecardCmd.Flags().String(scorecard.ProxyPullPolicyOpt, "Always", "Pull policy for scorecard proxy image") + scorecardCmd.Flags().String(scorecard.CRDsDirOpt, scaffold.CRDsDir, "Directory containing CRDs (all CRD manifest filenames must have the suffix 'crd.yaml')") + scorecardCmd.Flags().Bool(scorecard.VerboseOpt, false, "Enable verbose logging") if err := viper.BindPFlags(scorecardCmd.Flags()); err != nil { log.Fatalf("Failed to bind scorecard flags to viper: %v", err) diff --git a/cmd/operator-sdk/scorecard/test_definitions.go b/cmd/operator-sdk/scorecard/test_definitions.go deleted file mode 100644 index 216e1a6752d..00000000000 --- a/cmd/operator-sdk/scorecard/test_definitions.go +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2019 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorecard - -import ( - "context" - - olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Type Definitions - -// Test provides methods for running scorecard tests -type Test interface { - GetName() string - GetDescription() string - IsCumulative() bool - Run(context.Context) *TestResult -} - -// TestResult contains a test's points, suggestions, and errors -type TestResult struct { - Test Test - EarnedPoints int - MaximumPoints int - Suggestions []string - Errors []error -} - -// TestInfo contains information about the scorecard test -type TestInfo struct { - Name string - Description string - // If a test is set to cumulative, the scores of multiple runs of the same test on separate CRs are added together for the total score. - // If cumulative is false, if any test failed, the total score is 0/1. Otherwise 1/1. - Cumulative bool -} - -// GetName return the test name -func (i TestInfo) GetName() string { return i.Name } - -// GetDescription returns the test description -func (i TestInfo) GetDescription() string { return i.Description } - -// IsCumulative returns true if the test's scores are intended to be cumulative -func (i TestInfo) IsCumulative() bool { return i.Cumulative } - -// BasicTestConfig contains all variables required by the BasicTest TestSuite -type BasicTestConfig struct { - Client client.Client - CR *unstructured.Unstructured - ProxyPod *v1.Pod -} - -// OLMTestConfig contains all variables required by the OLMTest TestSuite -type OLMTestConfig struct { - Client client.Client - CR *unstructured.Unstructured - CSV *olmapiv1alpha1.ClusterServiceVersion - CRDsDir string - ProxyPod *v1.Pod -} - -// TestSuite contains a list of tests and results, along with the relative weights of each test -type TestSuite struct { - TestInfo - Tests []Test - TestResults []*TestResult - Weights map[string]float64 -} - -// Test definitions - -// CheckSpecTest is a scorecard test that verifies that the CR has a spec block -type CheckSpecTest struct { - TestInfo - BasicTestConfig -} - -// NewCheckSpecTest returns a new CheckSpecTest object -func NewCheckSpecTest(conf BasicTestConfig) *CheckSpecTest { - return &CheckSpecTest{ - BasicTestConfig: conf, - TestInfo: TestInfo{ - Name: "Spec Block Exists", - Description: "Custom Resource has a Spec Block", - Cumulative: false, - }, - } -} - -// CheckStatusTest is a scorecard test that verifies that the CR has a status block -type CheckStatusTest struct { - TestInfo - BasicTestConfig -} - -// NewCheckStatusTest returns a new CheckStatusTest object -func NewCheckStatusTest(conf BasicTestConfig) *CheckStatusTest { - return &CheckStatusTest{ - BasicTestConfig: conf, - TestInfo: TestInfo{ - Name: "Status Block Exists", - Description: "Custom Resource has a Status Block", - Cumulative: false, - }, - } -} - -// WritingIntoCRsHasEffectTest is a scorecard test that verifies that the operator is making PUT and/or POST requests to the API server -type WritingIntoCRsHasEffectTest struct { - TestInfo - BasicTestConfig -} - -// NewWritingIntoCRsHasEffectTest returns a new WritingIntoCRsHasEffectTest object -func NewWritingIntoCRsHasEffectTest(conf BasicTestConfig) *WritingIntoCRsHasEffectTest { - return &WritingIntoCRsHasEffectTest{ - BasicTestConfig: conf, - TestInfo: TestInfo{ - Name: "Writing into CRs has an effect", - Description: "A CR sends PUT/POST requests to the API server to modify resources in response to spec block changes", - Cumulative: false, - }, - } -} - -// CRDsHaveValidationTest is a scorecard test that verifies that all CRDs have a validation section -type CRDsHaveValidationTest struct { - TestInfo - OLMTestConfig -} - -// NewCRDsHaveValidationTest returns a new CRDsHaveValidationTest object -func NewCRDsHaveValidationTest(conf OLMTestConfig) *CRDsHaveValidationTest { - return &CRDsHaveValidationTest{ - OLMTestConfig: conf, - TestInfo: TestInfo{ - Name: "Provided APIs have validation", - Description: "All CRDs have an OpenAPI validation subsection", - Cumulative: true, - }, - } -} - -// CRDsHaveResourcesTest is a scorecard test that verifies that the CSV lists used resources in its owned CRDs secyion -type CRDsHaveResourcesTest struct { - TestInfo - OLMTestConfig -} - -// NewCRDsHaveResourcesTest returns a new CRDsHaveResourcesTest object -func NewCRDsHaveResourcesTest(conf OLMTestConfig) *CRDsHaveResourcesTest { - return &CRDsHaveResourcesTest{ - OLMTestConfig: conf, - TestInfo: TestInfo{ - Name: "Owned CRDs have resources listed", - Description: "All Owned CRDs contain a resources subsection", - Cumulative: true, - }, - } -} - -// AnnotationsContainExamplesTest is a scorecard test that verifies that the CSV contains examples via the alm-examples annotation -type AnnotationsContainExamplesTest struct { - TestInfo - OLMTestConfig -} - -// NewAnnotationsContainExamplesTest returns a new AnnotationsContainExamplesTest object -func NewAnnotationsContainExamplesTest(conf OLMTestConfig) *AnnotationsContainExamplesTest { - return &AnnotationsContainExamplesTest{ - OLMTestConfig: conf, - TestInfo: TestInfo{ - Name: "CRs have at least 1 example", - Description: "The CSV's metadata contains an alm-examples section", - Cumulative: true, - }, - } -} - -// SpecDescriptorsTest is a scorecard test that verifies that all spec fields have descriptors -type SpecDescriptorsTest struct { - TestInfo - OLMTestConfig -} - -// NewSpecDescriptorsTest returns a new SpecDescriptorsTest object -func NewSpecDescriptorsTest(conf OLMTestConfig) *SpecDescriptorsTest { - return &SpecDescriptorsTest{ - OLMTestConfig: conf, - TestInfo: TestInfo{ - Name: "Spec fields with descriptors", - Description: "All spec fields have matching descriptors in the CSV", - Cumulative: true, - }, - } -} - -// StatusDescriptorsTest is a scorecard test that verifies that all status fields have descriptors -type StatusDescriptorsTest struct { - TestInfo - OLMTestConfig -} - -// NewStatusDescriptorsTest returns a new StatusDescriptorsTest object -func NewStatusDescriptorsTest(conf OLMTestConfig) *StatusDescriptorsTest { - return &StatusDescriptorsTest{ - OLMTestConfig: conf, - TestInfo: TestInfo{ - Name: "Status fields with descriptors", - Description: "All status fields have matching descriptors in the CSV", - Cumulative: true, - }, - } -} - -// Test Suite Declarations - -// NewBasicTestSuite returns a new TestSuite object containing basic, functional operator tests -func NewBasicTestSuite(conf BasicTestConfig) *TestSuite { - ts := NewTestSuite( - "Basic Tests", - "Test suite that runs basic, functional operator tests", - ) - ts.AddTest(NewCheckSpecTest(conf), 1.5) - ts.AddTest(NewCheckStatusTest(conf), 1) - ts.AddTest(NewWritingIntoCRsHasEffectTest(conf), 1) - - return ts -} - -// NewOLMTestSuite returns a new TestSuite object containing CSV best practice checks -func NewOLMTestSuite(conf OLMTestConfig) *TestSuite { - ts := NewTestSuite( - "OLM Tests", - "Test suite checks if an operator's CSV follows best practices", - ) - - ts.AddTest(NewCRDsHaveValidationTest(conf), 1.25) - ts.AddTest(NewCRDsHaveResourcesTest(conf), 1) - ts.AddTest(NewAnnotationsContainExamplesTest(conf), 1) - ts.AddTest(NewSpecDescriptorsTest(conf), 1) - ts.AddTest(NewStatusDescriptorsTest(conf), 1) - - return ts -} - -// Helper functions - -// ResultsPassFail will be used when multiple CRs are supported -func ResultsPassFail(results []TestResult) (earned, max int) { - for _, result := range results { - if result.EarnedPoints != result.MaximumPoints { - return 0, 1 - } - } - return 1, 1 -} - -// ResultsCumulative will be used when multiple CRs are supported -func ResultsCumulative(results []TestResult) (earned, max int) { - for _, result := range results { - earned += result.EarnedPoints - max += result.MaximumPoints - } - return earned, max -} - -// AddTest adds a new Test to a TestSuite along with a relative weight for the new Test -func (ts *TestSuite) AddTest(t Test, weight float64) { - ts.Tests = append(ts.Tests, t) - ts.Weights[t.GetName()] = weight -} - -// TotalScore calculates and returns the total score of all run Tests in a TestSuite -func (ts *TestSuite) TotalScore() (score int) { - floatScore := 0.0 - for _, result := range ts.TestResults { - if result.MaximumPoints != 0 { - floatScore += (float64(result.EarnedPoints) / float64(result.MaximumPoints)) * ts.Weights[result.Test.GetName()] - } - } - // scale to a percentage - addedWeights := 0.0 - for _, weight := range ts.Weights { - addedWeights += weight - } - floatScore = floatScore * (100 / addedWeights) - return int(floatScore) -} - -// Run runs all Tests in a TestSuite -func (ts *TestSuite) Run(ctx context.Context) { - for _, test := range ts.Tests { - ts.TestResults = append(ts.TestResults, test.Run(ctx)) - } -} - -// NewTestSuite returns a new TestSuite with a given name and description -func NewTestSuite(name, description string) *TestSuite { - return &TestSuite{ - TestInfo: TestInfo{ - Name: name, - Description: description, - }, - Weights: make(map[string]float64), - } -} diff --git a/cmd/operator-sdk/scorecard/basic_tests.go b/internal/pkg/scorecard/basic_tests.go similarity index 53% rename from cmd/operator-sdk/scorecard/basic_tests.go rename to internal/pkg/scorecard/basic_tests.go index 53339cc73e8..6d4c7d98647 100644 --- a/cmd/operator-sdk/scorecard/basic_tests.go +++ b/internal/pkg/scorecard/basic_tests.go @@ -20,9 +20,90 @@ import ( "fmt" "strings" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) +// BasicTestConfig contains all variables required by the BasicTest TestSuite +type BasicTestConfig struct { + Client client.Client + CR *unstructured.Unstructured + ProxyPod *v1.Pod +} + +// Test Defintions + +// CheckSpecTest is a scorecard test that verifies that the CR has a spec block +type CheckSpecTest struct { + TestInfo + BasicTestConfig +} + +// NewCheckSpecTest returns a new CheckSpecTest object +func NewCheckSpecTest(conf BasicTestConfig) *CheckSpecTest { + return &CheckSpecTest{ + BasicTestConfig: conf, + TestInfo: TestInfo{ + Name: "Spec Block Exists", + Description: "Custom Resource has a Spec Block", + Cumulative: false, + }, + } +} + +// CheckStatusTest is a scorecard test that verifies that the CR has a status block +type CheckStatusTest struct { + TestInfo + BasicTestConfig +} + +// NewCheckStatusTest returns a new CheckStatusTest object +func NewCheckStatusTest(conf BasicTestConfig) *CheckStatusTest { + return &CheckStatusTest{ + BasicTestConfig: conf, + TestInfo: TestInfo{ + Name: "Status Block Exists", + Description: "Custom Resource has a Status Block", + Cumulative: false, + }, + } +} + +// WritingIntoCRsHasEffectTest is a scorecard test that verifies that the operator is making PUT and/or POST requests to the API server +type WritingIntoCRsHasEffectTest struct { + TestInfo + BasicTestConfig +} + +// NewWritingIntoCRsHasEffectTest returns a new WritingIntoCRsHasEffectTest object +func NewWritingIntoCRsHasEffectTest(conf BasicTestConfig) *WritingIntoCRsHasEffectTest { + return &WritingIntoCRsHasEffectTest{ + BasicTestConfig: conf, + TestInfo: TestInfo{ + Name: "Writing into CRs has an effect", + Description: "A CR sends PUT/POST requests to the API server to modify resources in response to spec block changes", + Cumulative: false, + }, + } +} + +// NewBasicTestSuite returns a new TestSuite object containing basic, functional operator tests +func NewBasicTestSuite(conf BasicTestConfig) *TestSuite { + ts := NewTestSuite( + "Basic Tests", + "Test suite that runs basic, functional operator tests", + ) + ts.AddTest(NewCheckSpecTest(conf), 1.5) + ts.AddTest(NewCheckStatusTest(conf), 1) + ts.AddTest(NewWritingIntoCRsHasEffectTest(conf), 1) + + return ts +} + +// Test Implementations + // Run - implements Test interface func (t *CheckSpecTest) Run(ctx context.Context) *TestResult { res := &TestResult{Test: t, MaximumPoints: 1} diff --git a/internal/pkg/scorecard/helpers.go b/internal/pkg/scorecard/helpers.go new file mode 100644 index 00000000000..f5bd0a9d04a --- /dev/null +++ b/internal/pkg/scorecard/helpers.go @@ -0,0 +1,37 @@ +// Copyright 2019 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +// These functions should be in the public test definitions file, but they are not complete/stable, +// so we'll keep these here until they get fully implemented + +// ResultsPassFail will be used when multiple CRs are supported +func ResultsPassFail(results []TestResult) (earned, max int) { + for _, result := range results { + if result.EarnedPoints != result.MaximumPoints { + return 0, 1 + } + } + return 1, 1 +} + +// ResultsCumulative will be used when multiple CRs are supported +func ResultsCumulative(results []TestResult) (earned, max int) { + for _, result := range results { + earned += result.EarnedPoints + max += result.MaximumPoints + } + return earned, max +} diff --git a/cmd/operator-sdk/scorecard/olm_tests.go b/internal/pkg/scorecard/olm_tests.go similarity index 73% rename from cmd/operator-sdk/scorecard/olm_tests.go rename to internal/pkg/scorecard/olm_tests.go index 252b2655fd7..9982229e13e 100644 --- a/cmd/operator-sdk/scorecard/olm_tests.go +++ b/internal/pkg/scorecard/olm_tests.go @@ -26,10 +26,113 @@ import ( log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) +// OLMTestConfig contains all variables required by the OLMTest TestSuite +type OLMTestConfig struct { + Client client.Client + CR *unstructured.Unstructured + CSV *olmapiv1alpha1.ClusterServiceVersion + CRDsDir string + ProxyPod *v1.Pod +} + +// Test Defintions + +// CRDsHaveValidationTest is a scorecard test that verifies that all CRDs have a validation section +type CRDsHaveValidationTest struct { + TestInfo + OLMTestConfig +} + +// NewCRDsHaveValidationTest returns a new CRDsHaveValidationTest object +func NewCRDsHaveValidationTest(conf OLMTestConfig) *CRDsHaveValidationTest { + return &CRDsHaveValidationTest{ + OLMTestConfig: conf, + TestInfo: TestInfo{ + Name: "Provided APIs have validation", + Description: "All CRDs have an OpenAPI validation subsection", + Cumulative: true, + }, + } +} + +// CRDsHaveResourcesTest is a scorecard test that verifies that the CSV lists used resources in its owned CRDs secyion +type CRDsHaveResourcesTest struct { + TestInfo + OLMTestConfig +} + +// NewCRDsHaveResourcesTest returns a new CRDsHaveResourcesTest object +func NewCRDsHaveResourcesTest(conf OLMTestConfig) *CRDsHaveResourcesTest { + return &CRDsHaveResourcesTest{ + OLMTestConfig: conf, + TestInfo: TestInfo{ + Name: "Owned CRDs have resources listed", + Description: "All Owned CRDs contain a resources subsection", + Cumulative: true, + }, + } +} + +// AnnotationsContainExamplesTest is a scorecard test that verifies that the CSV contains examples via the alm-examples annotation +type AnnotationsContainExamplesTest struct { + TestInfo + OLMTestConfig +} + +// NewAnnotationsContainExamplesTest returns a new AnnotationsContainExamplesTest object +func NewAnnotationsContainExamplesTest(conf OLMTestConfig) *AnnotationsContainExamplesTest { + return &AnnotationsContainExamplesTest{ + OLMTestConfig: conf, + TestInfo: TestInfo{ + Name: "CRs have at least 1 example", + Description: "The CSV's metadata contains an alm-examples section", + Cumulative: true, + }, + } +} + +// SpecDescriptorsTest is a scorecard test that verifies that all spec fields have descriptors +type SpecDescriptorsTest struct { + TestInfo + OLMTestConfig +} + +// NewSpecDescriptorsTest returns a new SpecDescriptorsTest object +func NewSpecDescriptorsTest(conf OLMTestConfig) *SpecDescriptorsTest { + return &SpecDescriptorsTest{ + OLMTestConfig: conf, + TestInfo: TestInfo{ + Name: "Spec fields with descriptors", + Description: "All spec fields have matching descriptors in the CSV", + Cumulative: true, + }, + } +} + +// StatusDescriptorsTest is a scorecard test that verifies that all status fields have descriptors +type StatusDescriptorsTest struct { + TestInfo + OLMTestConfig +} + +// NewStatusDescriptorsTest returns a new StatusDescriptorsTest object +func NewStatusDescriptorsTest(conf OLMTestConfig) *StatusDescriptorsTest { + return &StatusDescriptorsTest{ + OLMTestConfig: conf, + TestInfo: TestInfo{ + Name: "Status fields with descriptors", + Description: "All status fields have matching descriptors in the CSV", + Cumulative: true, + }, + } +} + func matchKind(kind1, kind2 string) bool { singularKind1, err := restMapper.ResourceSingularizer(kind1) if err != nil { @@ -44,6 +147,24 @@ func matchKind(kind1, kind2 string) bool { return strings.EqualFold(singularKind1, singularKind2) } +// NewOLMTestSuite returns a new TestSuite object containing CSV best practice checks +func NewOLMTestSuite(conf OLMTestConfig) *TestSuite { + ts := NewTestSuite( + "OLM Tests", + "Test suite checks if an operator's CSV follows best practices", + ) + + ts.AddTest(NewCRDsHaveValidationTest(conf), 1.25) + ts.AddTest(NewCRDsHaveResourcesTest(conf), 1) + ts.AddTest(NewAnnotationsContainExamplesTest(conf), 1) + ts.AddTest(NewSpecDescriptorsTest(conf), 1) + ts.AddTest(NewStatusDescriptorsTest(conf), 1) + + return ts +} + +// Test Implentations + // matchVersion checks if a CRD contains a specified version in a case insensitive manner func matchVersion(version string, crd *apiextv1beta1.CustomResourceDefinition) bool { if strings.EqualFold(version, crd.Spec.Version) { diff --git a/cmd/operator-sdk/scorecard/resource_handler.go b/internal/pkg/scorecard/resource_handler.go similarity index 100% rename from cmd/operator-sdk/scorecard/resource_handler.go rename to internal/pkg/scorecard/resource_handler.go index 2262dcf4f06..dc2fe27435d 100644 --- a/cmd/operator-sdk/scorecard/resource_handler.go +++ b/internal/pkg/scorecard/resource_handler.go @@ -26,11 +26,10 @@ import ( "github.com/operator-framework/operator-sdk/internal/util/yamlutil" proxyConf "github.com/operator-framework/operator-sdk/pkg/ansible/proxy/kubeconfig" "github.com/operator-framework/operator-sdk/pkg/k8sutil" - "github.com/spf13/viper" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/ghodss/yaml" log "github.com/sirupsen/logrus" + "github.com/spf13/viper" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +40,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" ) type cleanupFn func() error diff --git a/cmd/operator-sdk/scorecard/scorecard.go b/internal/pkg/scorecard/scorecard.go similarity index 98% rename from cmd/operator-sdk/scorecard/scorecard.go rename to internal/pkg/scorecard/scorecard.go index fe15b4d1706..f261e6ddccf 100644 --- a/cmd/operator-sdk/scorecard/scorecard.go +++ b/internal/pkg/scorecard/scorecard.go @@ -78,7 +78,6 @@ var ( deploymentName string proxyPodGlobal *v1.Pod cleanupFns []cleanupFn - ScorecardConf string ) const ( @@ -306,9 +305,10 @@ func ScorecardTests(cmd *cobra.Command, args []string) error { } func initConfig() error { - if ScorecardConf != "" { + // viper/cobra already has flags parsed at this point; we can check if a config file flag is set + if viper.GetString(ConfigOpt) != "" { // Use config file from the flag. - viper.SetConfigFile(ScorecardConf) + viper.SetConfigFile(viper.GetString(ConfigOpt)) } else { viper.AddConfigPath(projutil.MustGetwd()) // using SetConfigName allows users to use a .yaml, .json, or .toml file diff --git a/internal/pkg/scorecard/test_definitions.go b/internal/pkg/scorecard/test_definitions.go new file mode 100644 index 00000000000..b289e769685 --- /dev/null +++ b/internal/pkg/scorecard/test_definitions.go @@ -0,0 +1,107 @@ +// Copyright 2019 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "context" +) + +// Type Definitions + +// Test provides methods for running scorecard tests +type Test interface { + GetName() string + GetDescription() string + IsCumulative() bool + Run(context.Context) *TestResult +} + +// TestResult contains a test's points, suggestions, and errors +type TestResult struct { + Test Test + EarnedPoints int + MaximumPoints int + Suggestions []string + Errors []error +} + +// TestInfo contains information about the scorecard test +type TestInfo struct { + Name string + Description string + // If a test is set to cumulative, the scores of multiple runs of the same test on separate CRs are added together for the total score. + // If cumulative is false, if any test failed, the total score is 0/1. Otherwise 1/1. + Cumulative bool +} + +// GetName return the test name +func (i TestInfo) GetName() string { return i.Name } + +// GetDescription returns the test description +func (i TestInfo) GetDescription() string { return i.Description } + +// IsCumulative returns true if the test's scores are intended to be cumulative +func (i TestInfo) IsCumulative() bool { return i.Cumulative } + +// TestSuite contains a list of tests and results, along with the relative weights of each test +type TestSuite struct { + TestInfo + Tests []Test + TestResults []*TestResult + Weights map[string]float64 +} + +// Helper functions + +// AddTest adds a new Test to a TestSuite along with a relative weight for the new Test +func (ts *TestSuite) AddTest(t Test, weight float64) { + ts.Tests = append(ts.Tests, t) + ts.Weights[t.GetName()] = weight +} + +// TotalScore calculates and returns the total score of all run Tests in a TestSuite +func (ts *TestSuite) TotalScore() (score int) { + floatScore := 0.0 + for _, result := range ts.TestResults { + if result.MaximumPoints != 0 { + floatScore += (float64(result.EarnedPoints) / float64(result.MaximumPoints)) * ts.Weights[result.Test.GetName()] + } + } + // scale to a percentage + addedWeights := 0.0 + for _, weight := range ts.Weights { + addedWeights += weight + } + floatScore = floatScore * (100 / addedWeights) + return int(floatScore) +} + +// Run runs all Tests in a TestSuite +func (ts *TestSuite) Run(ctx context.Context) { + for _, test := range ts.Tests { + ts.TestResults = append(ts.TestResults, test.Run(ctx)) + } +} + +// NewTestSuite returns a new TestSuite with a given name and description +func NewTestSuite(name, description string) *TestSuite { + return &TestSuite{ + TestInfo: TestInfo{ + Name: name, + Description: description, + }, + Weights: make(map[string]float64), + } +}