diff --git a/changelog/fragments/scorecard-config-scaffold.yaml b/changelog/fragments/scorecard-config-scaffold.yaml new file mode 100644 index 0000000000..38038e0d18 --- /dev/null +++ b/changelog/fragments/scorecard-config-scaffold.yaml @@ -0,0 +1,9 @@ +entries: + - description: Implemented the scorecard config as a componentconfig object + kind: change + breaking: true + migration: + header: Update your scorecard config file to the new format + body: See the updated scorecard [config documentation](https://sdk.operatorframework.io/docs/scorecard/scorecard/#config-file) for details. + - description: Added `config/scorecard` kustomize scaffolds to `init` + kind: addition diff --git a/cmd/operator-sdk/generate/bundle/bundle.go b/cmd/operator-sdk/generate/bundle/bundle.go index db6639439a..10316ac937 100644 --- a/cmd/operator-sdk/generate/bundle/bundle.go +++ b/cmd/operator-sdk/generate/bundle/bundle.go @@ -35,6 +35,7 @@ import ( "github.com/operator-framework/operator-sdk/internal/registry" "github.com/operator-framework/operator-sdk/internal/scorecard" "github.com/operator-framework/operator-sdk/internal/util/projutil" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( @@ -218,6 +219,11 @@ func (c bundleCmd) runManifests(cfg *config.Config) (err error) { } } + // Write the scorecard config if it was passed. + if err := writeScorecardConfig(c.outputDir, col.ScorecardConfig); err != nil { + return fmt.Errorf("error writing bundle scorecard config: %v", err) + } + if !c.quiet && !c.stdout { fmt.Println("Bundle manifests generated successfully in", c.outputDir) } @@ -225,6 +231,25 @@ func (c bundleCmd) runManifests(cfg *config.Config) (err error) { return nil } +// writeScorecardConfig writes cfg to dir at the hard-coded config path 'config.yaml'. +func writeScorecardConfig(dir string, cfg v1alpha3.Configuration) error { + if cfg.Metadata.Name == "" { + return nil + } + + b, err := yaml.Marshal(cfg) + if err != nil { + return err + } + + cfgDir := filepath.Join(dir, filepath.FromSlash(scorecard.DefaultConfigDir)) + if err := os.MkdirAll(cfgDir, 0755); err != nil { + return err + } + scorecardConfigPath := filepath.Join(cfgDir, scorecard.ConfigFileName) + return ioutil.WriteFile(scorecardConfigPath, b, 0666) +} + // validateMetadata validates c for bundle metadata generation. func (c bundleCmd) validateMetadata(*config.Config) (err error) { // Ensure a default channel is present. @@ -282,6 +307,8 @@ func (c bundleCmd) generateMetadata(cfg *config.Config, manifestsDir, outputDir return nil } +// TODO(estroz): these updates need to be atomic because the bundle's Dockerfile and annotations.yaml +// cannot be out-of-sync. func updateMetadata(cfg *config.Config, bundleRoot string) error { bundleLabels := metricsannotations.MakeBundleMetadataLabels(cfg) for key, value := range scorecardannotations.MakeBundleMetadataLabels(scorecard.DefaultConfigDir) { @@ -292,8 +319,6 @@ func updateMetadata(cfg *config.Config, bundleRoot string) error { } // Write labels to bundle Dockerfile. - // NB(estroz): these "rewrites" need to be atomic because the bundle's Dockerfile and annotations.yaml - // cannot be out-of-sync. if err := rewriteDockerfileLabels(bundle.DockerFile, bundleLabels); err != nil { return fmt.Errorf("error writing LABEL's in %s: %v", bundle.DockerFile, err) } @@ -303,7 +328,8 @@ func updateMetadata(cfg *config.Config, bundleRoot string) error { // Add a COPY for the scorecard config to bundle Dockerfile. // TODO: change input config path to be a flag-based value. - err := writeDockerfileCOPYScorecardConfig(bundle.DockerFile, filepath.FromSlash(scorecard.DefaultConfigDir)) + localScorecardConfigPath := filepath.Join(bundleRoot, filepath.FromSlash(scorecard.DefaultConfigDir)) + err := writeDockerfileCOPYScorecardConfig(bundle.DockerFile, localScorecardConfigPath) if err != nil { return fmt.Errorf("error writing scorecard config COPY in %s: %v", bundle.DockerFile, err) } diff --git a/cmd/operator-sdk/generate/kustomize/manifests.go b/cmd/operator-sdk/generate/kustomize/manifests.go index dd60d01b0e..53e2e21f48 100644 --- a/cmd/operator-sdk/generate/kustomize/manifests.go +++ b/cmd/operator-sdk/generate/kustomize/manifests.go @@ -153,6 +153,7 @@ func (c *manifestsCmd) setDefaults(cfg *config.Config) { const manifestsKustomization = `resources: - ../default - ../samples +- ../scorecard ` // run generates kustomize bundle bases and a kustomization.yaml if one does not exist. diff --git a/internal/generate/collector/collect.go b/internal/generate/collector/collect.go index 0e672b7d24..77f26674f6 100644 --- a/internal/generate/collector/collect.go +++ b/internal/generate/collector/collect.go @@ -16,6 +16,7 @@ package collector import ( "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -32,6 +33,7 @@ import ( "sigs.k8s.io/yaml" "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + scorecardv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) // Manifests holds a collector of all manifests relevant to CSV updates. @@ -44,7 +46,9 @@ type Manifests struct { ValidatingWebhooks []admissionregv1.ValidatingWebhook MutatingWebhooks []admissionregv1.MutatingWebhook CustomResources []unstructured.Unstructured - Others []unstructured.Unstructured + ScorecardConfig scorecardv1alpha3.Configuration + + Others []unstructured.Unstructured } // UpdateFromDirs adds Roles, ClusterRoles, Deployments, and Custom Resource examples @@ -85,6 +89,10 @@ func (c *Manifests) UpdateFromDirs(deployDir, crdsDir string) error { err = c.addValidatingWebhookConfigurations(manifest) case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) + case scorecardv1alpha3.ConfigurationKind: + if gvk.GroupVersion() == scorecardv1alpha3.SchemeGroupVersion { + err = c.addScorecardConfig(manifest) + } default: err = c.addOthers(manifest) } @@ -145,6 +153,10 @@ func (c *Manifests) UpdateFromReader(r io.Reader) error { err = c.addValidatingWebhookConfigurations(manifest) case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) + case scorecardv1alpha3.ConfigurationKind: + if gvk.GroupVersion() == scorecardv1alpha3.SchemeGroupVersion { + err = c.addScorecardConfig(manifest) + } default: err = c.addOthers(manifest) } @@ -167,7 +179,7 @@ func (c *Manifests) UpdateFromReader(r io.Reader) error { return nil } -// addRoles assumes add manifest data in rawManifests are Roles and adds them +// addRoles assumes all manifest data in rawManifests are Roles and adds them // to the collector. func (c *Manifests) addRoles(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { @@ -180,7 +192,7 @@ func (c *Manifests) addRoles(rawManifests ...[]byte) error { return nil } -// addClusterRoles assumes add manifest data in rawManifests are ClusterRoles +// addClusterRoles assumes all manifest data in rawManifests are ClusterRoles // and adds them to the collector. func (c *Manifests) addClusterRoles(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { @@ -193,7 +205,7 @@ func (c *Manifests) addClusterRoles(rawManifests ...[]byte) error { return nil } -// addDeployments assumes add manifest data in rawManifests are Deployments +// addDeployments assumes all manifest data in rawManifests are Deployments // and adds them to the collector. func (c *Manifests) addDeployments(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { @@ -206,7 +218,7 @@ func (c *Manifests) addDeployments(rawManifests ...[]byte) error { return nil } -// addCustomResourceDefinitions assumes add manifest data in rawManifests are +// addCustomResourceDefinitions assumes all manifest data in rawManifests are // CustomResourceDefinitions and adds them to the collector. version determines // which CustomResourceDefinition type is used for all manifests in rawManifests. func (c *Manifests) addCustomResourceDefinitions(version string, rawManifests ...[]byte) (err error) { @@ -257,7 +269,21 @@ func (c *Manifests) addMutatingWebhookConfigurations(rawManifests ...[]byte) err return nil } -// addOthers assumes add manifest data in rawManifests are able to be +// addScorecardConfig assumes manifest data in rawManifests is a ScorecardConfigs and adds it to the collector. +// If a config has already been found, addScorecardConfig will return an error. +func (c *Manifests) addScorecardConfig(rawManifest []byte) error { + cfg := scorecardv1alpha3.Configuration{} + if err := yaml.Unmarshal(rawManifest, &cfg); err != nil { + return err + } + if c.ScorecardConfig.Metadata.Name != "" { + return errors.New("duplicate scorecard configurations in collector input") + } + c.ScorecardConfig = cfg + return nil +} + +// addOthers assumes all manifest data in rawManifests are able to be // unmarshalled into an Unstructured object and adds them to the collector. func (c *Manifests) addOthers(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { diff --git a/internal/plugins/golang/v2/init.go b/internal/plugins/golang/v2/init.go index 1e00261d55..466d27ba66 100644 --- a/internal/plugins/golang/v2/init.go +++ b/internal/plugins/golang/v2/init.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/kubebuilder/pkg/model/config" "sigs.k8s.io/kubebuilder/pkg/plugin" + "github.com/operator-framework/operator-sdk/internal/plugins/scorecard" utilplugins "github.com/operator-framework/operator-sdk/internal/util/plugins" ) @@ -50,6 +51,11 @@ func (p *initPlugin) Run() error { return err } + // Run the scorecard "phase 2" plugin. + if err := scorecard.RunInit(p.config); err != nil { + return err + } + // Update plugin config section with this plugin's configuration. cfg := Config{} if err := p.config.EncodePluginConfig(pluginConfigKey, cfg); err != nil { diff --git a/internal/plugins/helm/v1/init.go b/internal/plugins/helm/v1/init.go index eddd889ad4..f60ea76707 100644 --- a/internal/plugins/helm/v1/init.go +++ b/internal/plugins/helm/v1/init.go @@ -29,6 +29,7 @@ import ( "github.com/operator-framework/operator-sdk/internal/kubebuilder/cmdutil" "github.com/operator-framework/operator-sdk/internal/plugins/helm/v1/chartutil" "github.com/operator-framework/operator-sdk/internal/plugins/helm/v1/scaffolds" + "github.com/operator-framework/operator-sdk/internal/plugins/scorecard" utilplugins "github.com/operator-framework/operator-sdk/internal/util/plugins" ) @@ -128,7 +129,16 @@ func (p *initPlugin) InjectConfig(c *config.Config) { // Run will call the plugin actions func (p *initPlugin) Run() error { - return cmdutil.Run(p) + if err := cmdutil.Run(p); err != nil { + return err + } + + // Run the scorecard "phase 2" plugin. + if err := scorecard.RunInit(p.config); err != nil { + return err + } + + return nil } // Validate perform the required validations for this plugin diff --git a/internal/plugins/scorecard/init.go b/internal/plugins/scorecard/init.go new file mode 100644 index 0000000000..b2657edd35 --- /dev/null +++ b/internal/plugins/scorecard/init.go @@ -0,0 +1,267 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "text/template" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/kubebuilder/pkg/model/config" + "sigs.k8s.io/yaml" + + "github.com/operator-framework/operator-sdk/internal/scaffold/kustomize" + "github.com/operator-framework/operator-sdk/internal/scorecard" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" +) + +const ( + // kustomization.yaml file template for the scorecard componentconfig. This should always be written to + // config/scorecard/kustomization.yaml since it only references files in config. + scorecardKustomizationTemplate = `resources: +{{- range $i, $path := .ResourcePaths }} +- {{ $path }} +{{- end }} +patchesJson6902: +{{- range $i, $patch := .JSONPatches }} +- path: {{ $patch.Path }} + target: + group: {{ $patch.Target.Group }} + version: {{ $patch.Target.Version }} + kind: {{ $patch.Target.Kind }} + name: {{ $patch.Target.Name }} +{{- end }} +` + + // YAML file fragment to append to kustomization.yaml files. + kubebuilderScaffoldMarkerFragment = "# +kubebuilder:scaffold:patchesJson6902\n" +) + +const ( + // defaultTestImageTag points to the latest-released image. + // TODO: change the tag to "latest" once config scaffolding is in a release, + // as the new config spec won't work with the current latest image. + defaultTestImageTag = "quay.io/operator-framework/scorecard-test:master" + + // defaultConfigName is the default scorecard componentconfig's metadata.name, + // which must be set on all kustomize-able bases. This name is only used for + // `kustomize build` pattern match and not for on-cluster creation. + defaultConfigName = "config" +) + +// defaultDir is the default directory in which to generate kustomize bases and the kustomization.yaml. +var defaultDir = filepath.Join("config", "scorecard") + +// RunInit scaffolds kustomize files for kustomizing a scorecard componentconfig. +func RunInit(*config.Config) error { + return generate(defaultTestImageTag, defaultDir) +} + +// scorecardKustomizationValues holds data required to generate a scorecard's kustomization.yaml. +type scorecardKustomizationValues struct { + ResourcePaths []string + JSONPatches []kustomizationJSON6902Patch +} + +// kustomizationJSON6902Patch holds path and target data to write a patchesJson6902 list in a kustomization.yaml. +type kustomizationJSON6902Patch struct { + Path string + Target patchTarget +} + +// patchTarget holds target data for a kustomize patch. +type patchTarget struct { + schema.GroupVersionKind + Name string +} + +// generate scaffolds kustomize bundle bases and a kustomization.yaml. +// TODO(estroz): refactor this to be testable (in-mem fs) and easier to read. +func generate(testImageTag, outputDir string) error { + + kustomizationValues := scorecardKustomizationValues{} + + // Config bases. + basesDir := filepath.Join(outputDir, "bases") + if err := os.MkdirAll(basesDir, 0755); err != nil { + return err + } + + configBase := newConfigurationBase(defaultConfigName) + b, err := yaml.Marshal(configBase) + if err != nil { + return fmt.Errorf("error marshaling default config: %v", err) + } + relBasePath := filepath.Join("bases", scorecard.ConfigFileName) + basePath := filepath.Join(basesDir, scorecard.ConfigFileName) + if err := ioutil.WriteFile(basePath, b, 0666); err != nil { + return fmt.Errorf("error writing default scorecard config: %v", err) + } + kustomizationValues.ResourcePaths = append(kustomizationValues.ResourcePaths, relBasePath) + scorecardConfigTarget := patchTarget{ + GroupVersionKind: v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ConfigurationKind), + Name: defaultConfigName, + } + + // Config patches. + patchesDir := filepath.Join(outputDir, "patches") + if err := os.MkdirAll(patchesDir, 0755); err != nil { + return err + } + + // Basic scorecard tests patch. + basicPatch := newBasicConfigurationPatch(testImageTag) + b, err = yaml.Marshal(basicPatch) + if err != nil { + return fmt.Errorf("error marshaling basic patch config: %v", err) + } + basicPatchFileName := fmt.Sprintf("basic.%s", scorecard.ConfigFileName) + if err := ioutil.WriteFile(filepath.Join(patchesDir, basicPatchFileName), b, 0666); err != nil { + return fmt.Errorf("error writing basic scorecard config patch: %v", err) + } + kustomizationValues.JSONPatches = append(kustomizationValues.JSONPatches, kustomizationJSON6902Patch{ + Path: filepath.Join("patches", basicPatchFileName), + Target: scorecardConfigTarget, + }) + + // OLM scorecard tests patch. + olmPatch := newOLMConfigurationPatch(testImageTag) + b, err = yaml.Marshal(olmPatch) + if err != nil { + return fmt.Errorf("error marshaling OLM patch config: %v", err) + } + olmPatchFileName := fmt.Sprintf("olm.%s", scorecard.ConfigFileName) + if err := ioutil.WriteFile(filepath.Join(patchesDir, olmPatchFileName), b, 0666); err != nil { + return fmt.Errorf("error writing default scorecard config: %v", err) + } + kustomizationValues.JSONPatches = append(kustomizationValues.JSONPatches, kustomizationJSON6902Patch{ + Path: filepath.Join("patches", olmPatchFileName), + Target: scorecardConfigTarget, + }) + + // Write a kustomization.yaml to outputDir if one does not exist. + t, err := template.New("scorecard").Parse(scorecardKustomizationTemplate) + if err != nil { + return fmt.Errorf("error parsing default kustomize template: %v", err) + } + buf := bytes.Buffer{} + if err = t.Execute(&buf, kustomizationValues); err != nil { + return fmt.Errorf("error executing on default kustomize template: %v", err) + } + // Append the kubebuilder scaffold marker to make updates to this file in the future. + buf.Write([]byte(kubebuilderScaffoldMarkerFragment)) + if err := kustomize.Write(outputDir, buf.String()); err != nil { + return fmt.Errorf("error writing default scorecard kustomization.yaml: %v", err) + } + + return nil +} + +// jsonPatches is a list of JSON patch objects. +type jsonPatches []jsonPatchObject + +// jsonPatchObject is a JSON 6902 patch object specific to the scorecard's test configuration. +// https://kubernetes-sigs.github.io/kustomize/api-reference/kustomization/patchesjson6902/ for details. +type jsonPatchObject struct { + Op string `json:"op"` + Path string `json:"path"` + Value v1alpha3.TestConfiguration `json:"value"` +} + +// newConfigurationBase returns a scorecard componentconfig object with one parallel stage. +// The returned object is intended to be marshaled and written to disk as a kustomize base. +func newConfigurationBase(configName string) (cfg v1alpha3.Configuration) { + cfg.SetGroupVersionKind(v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ConfigurationKind)) + cfg.Metadata.Name = configName + cfg.Stages = []v1alpha3.StageConfiguration{ + { + Parallel: true, + Tests: []v1alpha3.TestConfiguration{}, + }, + } + return cfg +} + +const defaultJSONPath = "/stages/0/tests/-" + +// newBasicConfigurationPatch returns default "basic" test configurations as JSON patch objects +// to be inserted into the componentconfig base as a first stage test element. +// The returned patches are intended to be marshaled and written to disk as in a kustomize patch file. +func newBasicConfigurationPatch(testImageTag string) (ps jsonPatches) { + for _, cfg := range makeDefaultBasicTestConfigs(testImageTag) { + ps = append(ps, jsonPatchObject{ + Op: "add", + Path: defaultJSONPath, + Value: cfg, + }) + } + return ps +} + +// makeDefaultBasicTestConfigs returns all default "basic" test configurations. +func makeDefaultBasicTestConfigs(testImageTag string) (cfgs []v1alpha3.TestConfiguration) { + for _, testName := range []string{"basic-check-spec"} { + cfgs = append(cfgs, v1alpha3.TestConfiguration{ + Image: testImageTag, + Entrypoint: []string{"scorecard-test", testName}, + Labels: map[string]string{ + "suite": "basic", + "test": fmt.Sprintf("%s-test", testName), + }, + }) + } + + return cfgs +} + +// newOLMConfigurationPatch returns default "olm" test configurations as JSON patch objects +// to be inserted into the componentconfig base as a first stage test element. +// The returned patches are intended to be marshaled and written to disk as in a kustomize patch file. +func newOLMConfigurationPatch(testImageTag string) (ps jsonPatches) { + for _, cfg := range makeDefaultOLMTestConfigs(testImageTag) { + ps = append(ps, jsonPatchObject{ + Op: "add", + Path: defaultJSONPath, + Value: cfg, + }) + } + return ps +} + +// makeDefaultOLMTestConfigs returns all default "olm" test configurations. +func makeDefaultOLMTestConfigs(testImageTag string) (cfgs []v1alpha3.TestConfiguration) { + for _, testName := range []string{ + "olm-bundle-validation", + "olm-crds-have-validation", + "olm-crds-have-resources", + "olm-spec-descriptors", + "olm-status-descriptors"} { + + cfgs = append(cfgs, v1alpha3.TestConfiguration{ + Image: testImageTag, + Entrypoint: []string{"scorecard-test", testName}, + Labels: map[string]string{ + "suite": "olm", + "test": fmt.Sprintf("%s-test", testName), + }, + }) + } + + return cfgs +} diff --git a/internal/scorecard/config.go b/internal/scorecard/config.go index 9854b70abe..cf1a74fa33 100644 --- a/internal/scorecard/config.go +++ b/internal/scorecard/config.go @@ -18,6 +18,8 @@ import ( "io/ioutil" "sigs.k8s.io/yaml" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( @@ -27,43 +29,20 @@ const ( DefaultConfigDir = "tests/scorecard/" ) -type Stage struct { - Parallel bool `yaml:"parallel"` - Tests []Test `yaml:"tests"` -} - -type Test struct { - // Image is the name of the testimage - Image string `json:"image"` - // Entrypoint is list of commands and arguments passed to the test image - Entrypoint []string `json:"entrypoint,omitempty"` - // Labels that further describe the test and enable selection - Labels map[string]string `json:"labels,omitempty"` -} - -// Config represents the set of test configurations which scorecard -// would run based on user input -type Config struct { - Stages []Stage `yaml:"stages"` -} - // LoadConfig will find and return the scorecard config, the config file // is found from a bundle location (TODO bundle image) // scorecard config.yaml is expected to be in the bundle at the following // location: tests/scorecard/config.yaml // the user can override this location using the --config CLI flag -func LoadConfig(configFilePath string) (Config, error) { - c := Config{} +// TODO: version this. +func LoadConfig(configFilePath string) (v1alpha3.Configuration, error) { + c := v1alpha3.Configuration{} - // TODO handle bundle images, not just on-disk yamlFile, err := ioutil.ReadFile(configFilePath) if err != nil { return c, err } - if err := yaml.Unmarshal(yamlFile, &c); err != nil { - return c, err - } - - return c, nil + err = yaml.Unmarshal(yamlFile, &c) + return c, err } diff --git a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index 103a2ec1b4..9d34e5aebd 100644 --- a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -1,3 +1,7 @@ +kind: Configuration +apiversion: scorecard.operatorframework.io/v1alpha3 +metadata: + name: config stages: - tests: - image: quay.io/username/custom-scorecard-tests:dev diff --git a/internal/scorecard/formatting.go b/internal/scorecard/formatting.go index a6b7c0622a..96a7b79afb 100644 --- a/internal/scorecard/formatting.go +++ b/internal/scorecard/formatting.go @@ -46,11 +46,7 @@ func (o Scorecard) List() v1alpha3.TestList { tests := o.selectTests(stage) for _, test := range tests { item := v1alpha3.NewTest() - item.Spec = v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, - } + item.Spec = test output.Items = append(output.Items, item) } } diff --git a/internal/scorecard/labels_test.go b/internal/scorecard/labels_test.go index bbdf705072..6c6e31035b 100644 --- a/internal/scorecard/labels_test.go +++ b/internal/scorecard/labels_test.go @@ -17,8 +17,9 @@ package scorecard import ( "testing" - "gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/labels" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) func TestEmptySelector(t *testing.T) { @@ -26,25 +27,22 @@ func TestEmptySelector(t *testing.T) { cases := []struct { selectorValue string testsSelected int + config v1alpha3.Configuration wantError bool }{ - {"", 7, false}, - {"suite in (kuttl)", 1, false}, - {"test=basic-check-spec-test", 1, false}, - {"testXwriteintocr", 0, false}, - {"test X writeintocr", 0, true}, + {"", 7, testConfig, false}, + {"suite in (kuttl)", 1, testConfig, false}, + {"test=basic-check-spec-test", 1, testConfig, false}, + {"testXwriteintocr", 0, testConfig, false}, + {"test X writeintocr", 0, testConfig, true}, } for _, c := range cases { t.Run(c.selectorValue, func(t *testing.T) { o := Scorecard{} + o.Config = c.config - err := yaml.Unmarshal([]byte(testConfig), &o.Config) - if err != nil { - t.Log(err) - return - } - + var err error o.Selector, err = labels.Parse(c.selectorValue) if err == nil && c.wantError { t.Fatalf("Wanted error but got no error") @@ -65,52 +63,83 @@ func TestEmptySelector(t *testing.T) { } } -const testConfig = `stages: -- tests: - - image: quay.io/someuser/customtest1:v0.0.1 - entrypoint: - - custom-test - labels: - suite: custom - test: customtest1 - - image: quay.io/someuser/customtest2:v0.0.1 - entrypoint: - - custom-test - labels: - suite: custom - test: customtest2 - - image: quay.io/redhat/basictests:v0.0.1 - entrypoint: - - scorecard-test - - basic-check-spec - labels: - suite: basic - test: basic-check-spec-test - - image: quay.io/redhat/basictests:v0.0.1 - entrypoint: - - scorecard-test - - basic-check-status - labels: - suite: basic - test: basic-check-status-test - - image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: - - scorecard-test - - olm-bundle-validation - labels: - suite: olm - test: olm-bundle-validation-test - - image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: - - scorecard-test - - olm-crds-have-validation - labels: - suite: olm - test: olm-crds-have-validation-test - - image: quay.io/redhat/kuttltests:v0.0.1 - labels: - suite: kuttl - entrypoint: - - kuttl-test - - olm-status-descriptors -` +var testConfig = v1alpha3.Configuration{ + Stages: []v1alpha3.StageConfiguration{ + { + Tests: []v1alpha3.TestConfiguration{ + {Image: "quay.io/someuser/customtest1:v0.0.1", + Entrypoint: []string{ + "custom-test", + }, + Labels: map[string]string{ + "suite": "custom", + "test": "customtest1", + }, + }, + + {Image: "quay.io/someuser/customtest2:v0.0.1", + Entrypoint: []string{ + "custom-test", + }, + Labels: map[string]string{ + "suite": "custom", + "test": "customtest2", + }, + }, + + {Image: "quay.io/redhat/basictests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "basic-check-spec", + }, + Labels: map[string]string{ + "suite": "basic", + "test": "basic-check-spec-test", + }, + }, + + {Image: "quay.io/redhat/basictests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "basic-check-status", + }, + Labels: map[string]string{ + "suite": "basic", + "test": "basic-check-status-test", + }, + }, + + {Image: "quay.io/redhat/olmtests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "olm-bundle-validation", + }, + Labels: map[string]string{ + "suite": "olm", + "test": "olm-bundle-validation-test", + }, + }, + + {Image: "quay.io/redhat/olmtests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "olm-crds-have-validation", + }, + Labels: map[string]string{ + "suite": "olm", + "test": "olm-crds-have-validation-test", + }, + }, + {Image: "quay.io/redhat/kuttltests:v0.0.1", + Entrypoint: []string{ + "kuttl-test", + "olm-status-descriptors", + }, + Labels: map[string]string{ + "suite": "kuttl", + }, + }, + }, + }, + }, +} diff --git a/internal/scorecard/run_test.go b/internal/scorecard/run_test.go index 5a92aa16ce..52648b6415 100644 --- a/internal/scorecard/run_test.go +++ b/internal/scorecard/run_test.go @@ -153,11 +153,11 @@ func TestRunSequentialFail(t *testing.T) { func getFakeScorecard(parallel bool) Scorecard { return Scorecard{ - Config: Config{ - Stages: []Stage{ + Config: v1alpha3.Configuration{ + Stages: []v1alpha3.StageConfiguration{ { Parallel: parallel, - Tests: []Test{ + Tests: []v1alpha3.TestConfiguration{ {}, {}, }, diff --git a/internal/scorecard/scorecard.go b/internal/scorecard/scorecard.go index 0011a7f5c7..f1c9f87684 100644 --- a/internal/scorecard/scorecard.go +++ b/internal/scorecard/scorecard.go @@ -32,12 +32,12 @@ import ( type TestRunner interface { Initialize(context.Context) error - RunTest(context.Context, Test) (*v1alpha3.TestStatus, error) + RunTest(context.Context, v1alpha3.TestConfiguration) (*v1alpha3.TestStatus, error) Cleanup(context.Context) error } type Scorecard struct { - Config Config + Config v1alpha3.Configuration Selector labels.Selector TestRunner TestRunner SkipCleanup bool @@ -93,11 +93,11 @@ func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { return testOutput, nil } -func (o Scorecard) runStageParallel(ctx context.Context, tests []Test, results chan<- v1alpha3.Test) { +func (o Scorecard) runStageParallel(ctx context.Context, tests []v1alpha3.TestConfiguration, results chan<- v1alpha3.Test) { var wg sync.WaitGroup for _, t := range tests { wg.Add(1) - go func(test Test) { + go func(test v1alpha3.TestConfiguration) { results <- o.runTest(ctx, test) wg.Done() }(t) @@ -105,32 +105,28 @@ func (o Scorecard) runStageParallel(ctx context.Context, tests []Test, results c wg.Wait() } -func (o Scorecard) runStageSequential(ctx context.Context, tests []Test, results chan<- v1alpha3.Test) { +func (o Scorecard) runStageSequential(ctx context.Context, tests []v1alpha3.TestConfiguration, results chan<- v1alpha3.Test) { for _, test := range tests { results <- o.runTest(ctx, test) } } -func (o Scorecard) runTest(ctx context.Context, test Test) v1alpha3.Test { +func (o Scorecard) runTest(ctx context.Context, test v1alpha3.TestConfiguration) v1alpha3.Test { result, err := o.TestRunner.RunTest(ctx, test) if err != nil { result = convertErrorToStatus(err, "") } out := v1alpha3.NewTest() - out.Spec = v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, - } + out.Spec = test out.Status = *result return out } // selectTests applies an optionally passed selector expression // against the configured set of tests, returning the selected tests -func (o *Scorecard) selectTests(stage Stage) []Test { - selected := make([]Test, 0) +func (o *Scorecard) selectTests(stage v1alpha3.StageConfiguration) []v1alpha3.TestConfiguration { + selected := make([]v1alpha3.TestConfiguration, 0) for _, test := range stage.Tests { if o.Selector == nil || o.Selector.String() == "" || o.Selector.Matches(labels.Set(test.Labels)) { // TODO olm manifests check @@ -187,7 +183,7 @@ func (r PodTestRunner) Cleanup(ctx context.Context) (err error) { } // RunTest executes a single test -func (r PodTestRunner) RunTest(ctx context.Context, test Test) (*v1alpha3.TestStatus, error) { +func (r PodTestRunner) RunTest(ctx context.Context, test v1alpha3.TestConfiguration) (*v1alpha3.TestStatus, error) { // Create a Pod to run the test podDef := getPodDefinition(r.configMapName, test, r) pod, err := r.Client.CoreV1().Pods(r.Namespace).Create(ctx, podDef, metav1.CreateOptions{}) @@ -204,7 +200,7 @@ func (r PodTestRunner) RunTest(ctx context.Context, test Test) (*v1alpha3.TestSt } // RunTest executes a single test -func (r FakeTestRunner) RunTest(ctx context.Context, test Test) (result *v1alpha3.TestStatus, err error) { +func (r FakeTestRunner) RunTest(ctx context.Context, test v1alpha3.TestConfiguration) (result *v1alpha3.TestStatus, err error) { select { case <-time.After(r.Sleep): return r.TestStatus, r.Error diff --git a/internal/scorecard/testdata/bundle.tar.gz b/internal/scorecard/testdata/bundle.tar.gz index e0b6c54589..5f67c14017 100644 Binary files a/internal/scorecard/testdata/bundle.tar.gz and b/internal/scorecard/testdata/bundle.tar.gz differ diff --git a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml index 4831301bbc..6e693a5916 100644 --- a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml @@ -1,43 +1,47 @@ +kind: Configuration +apiversion: scorecard.operatorframework.io/v1alpha3 +metadata: + name: config stages: - parallel: true tests: - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: + entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: + entrypoint: - scorecard-test - olm-bundle-validation labels: suite: olm test: olm-bundle-validation-test - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: + entrypoint: - scorecard-test - olm-crds-have-validation labels: suite: olm test: olm-crds-have-validation-test - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: + entrypoint: - scorecard-test - olm-crds-have-resources labels: suite: olm test: olm-crds-have-resources-test - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: + entrypoint: - scorecard-test - olm-spec-descriptors labels: suite: olm test: olm-spec-descriptors-test - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: + entrypoint: - scorecard-test - olm-status-descriptors labels: diff --git a/internal/scorecard/testpod.go b/internal/scorecard/testpod.go index 41203465fc..3ccf5033e9 100644 --- a/internal/scorecard/testpod.go +++ b/internal/scorecard/testpod.go @@ -24,6 +24,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( @@ -33,7 +35,7 @@ const ( // getPodDefinition fills out a Pod definition based on // information from the test -func getPodDefinition(configMapName string, test Test, r PodTestRunner) *v1.Pod { +func getPodDefinition(configMapName string, test v1alpha3.TestConfiguration, r PodTestRunner) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("scorecard-test-%s", rand.String(4)), diff --git a/pkg/apis/scorecard/v1alpha3/configuration_types.go b/pkg/apis/scorecard/v1alpha3/configuration_types.go new file mode 100644 index 0000000000..631f46c8dc --- /dev/null +++ b/pkg/apis/scorecard/v1alpha3/configuration_types.go @@ -0,0 +1,55 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConfigurationKind is the default scorecard componentconfig kind. +const ConfigurationKind = "Configuration" + +// Configuration represents the set of test configurations which scorecard would run. +type Configuration struct { + metav1.TypeMeta `json:",inline" yaml:",inline"` + + // Do not use metav1.ObjectMeta because this "object" should not be treated as an actual object. + Metadata struct { + // Name is a required field for kustomize-able manifests, and is not used on-cluster (nor is the config itself). + Name string `json:"name,omitempty" yaml:"name,omitempty"` + } `json:"metadata,omitempty" yaml:"metadata,omitempty"` + + // Stages is a set of test stages to run. Once a stage is finished, the next stage in the slice will be run. + Stages []StageConfiguration `json:"stages" yaml:"stages"` +} + +// StageConfiguration configures a set of tests to be run. +type StageConfiguration struct { + // Parallel, if true, will run each test in tests in parallel. + // The default is to wait until a test finishes to run the next. + Parallel bool `json:"parallel,omitempty" yaml:"parallel,omitempty"` + // Tests are a list of tests to run. + Tests []TestConfiguration `json:"tests" yaml:"tests"` +} + +// TestConfiguration configures a specific scorecard test, identified by entrypoint. +type TestConfiguration struct { + // Image is the name of the test image. + Image string `json:"image" yaml:"image"` + // Entrypoint is a list of commands and arguments passed to the test image. + Entrypoint []string `json:"entrypoint,omitempty" yaml:"entrypoint,omitempty"` + // Labels further describe the test and enable selection. + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` +} diff --git a/pkg/apis/scorecard/v1alpha3/types.go b/pkg/apis/scorecard/v1alpha3/test_types.go similarity index 83% rename from pkg/apis/scorecard/v1alpha3/types.go rename to pkg/apis/scorecard/v1alpha3/test_types.go index 2096792e86..4480462ad5 100644 --- a/pkg/apis/scorecard/v1alpha3/types.go +++ b/pkg/apis/scorecard/v1alpha3/test_types.go @@ -30,16 +30,6 @@ const ( ErrorState State = "error" ) -// TestSpec contains the spec details of an individual scorecard test -type TestSpec struct { - // Image is the name of the testimage - Image string `json:"image"` - // Entrypoint is list of commands and arguments passed to the test image - Entrypoint []string `json:"entrypoint,omitempty"` - // Labels that further describe the test and enable selection - Labels map[string]string `json:"labels,omitempty"` -} - // TestResult contains the results of an individual scorecard test type TestResult struct { // Name is the name of the test @@ -64,8 +54,8 @@ type TestStatus struct { // Test specifies a single test run. type Test struct { metav1.TypeMeta `json:",inline"` - Spec TestSpec `json:"spec,omitempty"` - Status TestStatus `json:"status,omitempty"` + Spec TestConfiguration `json:"spec,omitempty"` + Status TestStatus `json:"status,omitempty"` } // TestList is a list of tests. diff --git a/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go index 0ba9be02e9..38350de867 100644 --- a/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go @@ -22,12 +22,61 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + out.Metadata = in.Metadata + if in.Stages != nil { + in, out := &in.Stages, &out.Stages + *out = make([]StageConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageConfiguration) DeepCopyInto(out *StageConfiguration) { + *out = *in + if in.Tests != nil { + in, out := &in.Tests, &out.Tests + *out = make([]TestConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageConfiguration. +func (in *StageConfiguration) DeepCopy() *StageConfiguration { + if in == nil { + return nil + } + out := new(StageConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Test) DeepCopyInto(out *Test) { *out = *in out.TypeMeta = in.TypeMeta in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Test. @@ -48,6 +97,34 @@ func (in *Test) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestConfiguration) DeepCopyInto(out *TestConfiguration) { + *out = *in + if in.Entrypoint != nil { + in, out := &in.Entrypoint, &out.Entrypoint + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestConfiguration. +func (in *TestConfiguration) DeepCopy() *TestConfiguration { + if in == nil { + return nil + } + out := new(TestConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestList) DeepCopyInto(out *TestList) { *out = *in @@ -59,6 +136,7 @@ func (in *TestList) DeepCopyInto(out *TestList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestList. @@ -84,6 +162,7 @@ func (in *TestResult) DeepCopyInto(out *TestResult) { *out = make([]string, len(*in)) copy(*out, *in) } + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestResult. @@ -96,33 +175,6 @@ func (in *TestResult) DeepCopy() *TestResult { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TestSpec) DeepCopyInto(out *TestSpec) { - *out = *in - if in.Entrypoint != nil { - in, out := &in.Entrypoint, &out.Entrypoint - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestSpec. -func (in *TestSpec) DeepCopy() *TestSpec { - if in == nil { - return nil - } - out := new(TestSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestStatus) DeepCopyInto(out *TestStatus) { *out = *in @@ -133,6 +185,7 @@ func (in *TestStatus) DeepCopyInto(out *TestStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestStatus. diff --git a/website/content/en/docs/advanced-topics/scorecard/custom-tests.md b/website/content/en/docs/advanced-topics/scorecard/custom-tests.md index 2ded308724..463791ff2b 100644 --- a/website/content/en/docs/advanced-topics/scorecard/custom-tests.md +++ b/website/content/en/docs/advanced-topics/scorecard/custom-tests.md @@ -3,7 +3,7 @@ title: Writing Custom Scorecard Tests weight: 50 --- -This guide outlines the steps which can be followed to extend the existing scorecard tests and implement operator specific custom tests. +This guide outlines the steps which can be followed to extend the existing scorecard tests and implement operator specific custom tests. ## Run scorecard with custom tests: @@ -14,6 +14,18 @@ The following steps explain creating of a custom test image which can be used wi The sample test image repository present [here][custom_scorecard_repo] has the following project structure: ``` +$ tree . +. +... +├── config +| ... +│   └── scorecard +│      ├── bases +│      │   └── config.yaml +│      ├── kustomization.yaml +│      └── patches +│      ├── basic.config.yaml +│      └── olm.config.yaml ├── Makefile ├── bundle │   ├── manifests @@ -41,10 +53,11 @@ The sample test image repository present [here][custom_scorecard_repo] has the f └── tests.go ``` +1. `config/scorecard` - Contains a kustomization for generating a config from a base and set of overlays. 1. `bundle/` - Contains bundle manifests and metadata under test. -2. `bundle/tests/scorecard/config.yaml` - Configuration yaml to define and run scorecard tests. -3. `images/custom-scorecard-tests/cmd/test/main.go` - Scorecard test binary. -4. `internal/tests/tests.go` - Contains the implementation of custom tests specific to the operator. +1. `bundle/tests/scorecard/config.yaml` - Configuration file generated by `make bundle` from the `config/scorecard` kustomization. +1. `images/custom-scorecard-tests/cmd/test/main.go` - Scorecard test binary. +1. `internal/tests/tests.go` - Contains the implementation of custom tests specific to the operator. #### Writing custom test logic: @@ -56,48 +69,73 @@ The `tests.go` file is where the custom tests are implemented in the sample test package tests import ( - "github.com/operator-framework/operator-registry/pkg/registry" - scapiv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" + "github.com/operator-framework/operator-registry/pkg/registry" + scapiv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( - CustomTest1Name = "customtest1" + CustomTest1Name = "customtest1" ) -// CustomTest1 +// CustomTest1 func CustomTest1(bundle registry.Bundle) scapiv1alpha3.TestStatus { - r := scapiv1alpha3.TestResult{} - r.Name = CustomTest1Name - r.Description = "Custom Test 1" - r.State = scapiv1alpha3.PassState - r.Errors = make([]string, 0) - r.Suggestions = make([]string, 0) + r := scapiv1alpha3.TestResult{} + r.Name = CustomTest1Name + r.Description = "Custom Test 1" + r.State = scapiv1alpha3.PassState + r.Errors = make([]string, 0) + r.Suggestions = make([]string, 0) // Implement relevant custom test logic here - return wrapResult(r) + return wrapResult(r) } ``` ### Scorecard Configuration file: -The [configuration file][config_yaml] includes test definitions and metadata to run the test. For the example `CustomTest1` function, the following fields should be specified in `config.yaml`. +The [configuration file][config_yaml] includes test definitions and metadata to run the test. +This file is constructed using a kustomization under `config/scorecard`, with overlays for test sets. + +For the example `CustomTest1` function, add the following to `config/scorecard/customtest1.config.yaml`. ```yaml -stages: -- tests: - - image: quay.io/username/custom-scorecard-tests:dev +- op: add + path: /stages/0/tests/- + value: + image: quay.io//custom-scorecard-tests:latest entrypoint: - custom-scorecard-tests - customtest1 labels: suite: custom test: customtest1 - ``` +``` The important fields to note here are: 1. `image` - name and tag of the test image which was specified in the Makefile. -2. `labels` - the name of the `test` and `suite` the test function belongs to. This can be specified in the `operator-sdk scorecard` command to run the desired test. +2. `labels` - the name of the `test` and `suite` the test function belongs to. +This can be specified in the `operator-sdk scorecard` command to run the desired test. + +Next, add a [JSON 6902 patch][kustomize-patchJson6902] to your `config/scorecard/kustomization.yaml`: + +```yaml +patchesJson6902: +... +- path: patches/customtest1.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +``` + +Once you run `make bundle`, the `bundle/tests/scorecard/config.yaml` will be (re)generated with your custom test. + +If generating a config file outside of the on-disk bundle, you can run: +```console +$ kustomize build config/scorecard > path/to/config.yaml +``` **Note**: The default location of `config.yaml` inside the bundle is `/tests/scorecard/config.yaml`. It can be overridden using the `--config` flag. For more details regarding the configuration file refer to [user docs][user_doc]. @@ -106,10 +144,10 @@ The important fields to note here are: The scorecard test image implementation requires the bundle under test to be present in the test image. The `apimanifests.GetBundleFromDir()` function reads the pod's bundle to fetch the manifests and scorecard configuration from desired path. ```Go - cfg, err := apimanifests.GetBundleFromDir(scorecard.PodBundleRoot) - if err != nil { - log.Fatal(err.Error()) - } +cfg, err := apimanifests.GetBundleFromDir(scorecard.PodBundleRoot) +if err != nil { + log.Fatal(err.Error()) +} ``` The scorecard binary uses `config.yaml` file to locate tests and execute the them as Pods which scorecard creates. Custom test images are included into Pods that scorecard creates, passing in the bundle contents on a shared mount point to the test image container. The specific custom test that is executed is driven by the config.yaml's entry-point command and arguments. @@ -118,11 +156,11 @@ An example scorecard binary is present [here][scorecard_binary]. The names with which the tests are identified in `config.yaml` and would be passed in the `scorecard` command, are to be specified here. ```Go -... +... switch entrypoint[0] { case tests.CustomTest1Name: - result = tests.CustomTest1(cfg) - ... + result = tests.CustomTest1(cfg) + ... } ... ``` @@ -131,9 +169,9 @@ The result of the custom tests which is in `scapiv1alpha3.TestResult` format, is ```Go prettyJSON, err := json.MarshalIndent(result, "", " ") - if err != nil { - log.Fatal("Failed to generate json", err) - } +if err != nil { + log.Fatal("Failed to generate json", err) +} fmt.Printf("%s\n", string(prettyJSON)) ``` @@ -141,11 +179,11 @@ The names of the custom tests are also included in `printValidTests()` function: ```Go func printValidTests() (result scapiv1alpha3.TestStatus) { -... - str := fmt.Sprintf("Valid tests for this image include: %s, - tests.CustomTest1Name - result.Errors = append(result.Errors, str") -... + ... + str := fmt.Sprintf("Valid tests for this image include: %s", tests.CustomTest1Name) + result.Errors = append(result.Errors, str) + ... +} ``` @@ -166,18 +204,34 @@ docker push //:tag The `operator-sdk scorecard` command is used to execute the scorecard tests by specifying the location of test bundle in the command. The name or suite of the tests which are to be executed can be specified with the `--selector` flag. The command will create scorecard pods with the image specified in `config.yaml` for the respective test. For example, the `CustomTest1Name` test provides the following json output. ```console -operator-sdk scorecard --selector=suite=custom -o json --wait-time=32s --skip-cleanup=false +$ operator-sdk scorecard --selector=suite=custom -o json --wait-time=32s --skip-cleanup=false { - "metadata": { - "creationTimestamp": null - }, - "log": "", - "results": [ + "kind": "TestList", + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "items": [ { - "name": "customtest1", - "description": "", - "state": "pass", - "log": "an ISV custom test" + "kind": "Test", + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "spec": { + "image": "quay.io/operator-framework/scorecard-test:latest", + "entrypoint": [ + "custom-scorecard-tests", + "customtest1" + ], + "labels": { + "suite": "custom", + "test": "customtest1" + } + }, + "status": { + "results": [ + { + "name": "customtest1", + "log": "an ISV custom test", + "state": "pass" + } + ] + } } ] } @@ -187,7 +241,7 @@ operator-sdk scorecard --selector=suite=custom -o json --w ### Debugging scorecard custom tests -The `--skip-cleanup` flag can be used when executing the `operator-sdk scorecard` command to cause the scorecard created test pods to be unremoved. +The `--skip-cleanup` flag can be used when executing the `operator-sdk scorecard` command to cause the scorecard created test pods to be unremoved. This is useful when debugging or writing new tests so that you can view the test logs or the pod manifests. @@ -208,14 +262,14 @@ namespaces for your test but instead considers these resources to be outside its scope. You can however implement whatever service accounts your tests require and then specify that service account from the command line using the service-account flag: -``` -operator-sdk scorecard --service-account=mycustomsa +```console +$ operator-sdk scorecard --service-account=mycustomsa ``` Also, you can set up a non-default namespace that your tests will be executed within using the following namespace flag: -``` -operator-sdk scorecard --namespace=mycustomns +```console +$ operator-sdk scorecard --namespace=mycustomns ``` If you do not specify either of these flags, the default namespace @@ -231,9 +285,9 @@ https://github.com/operator-framework/operator-sdk/blob/master/pkg/apis/scorecar ### Accessing the Kube API Within your custom tests you might require connecting to the Kube API. -In golang, you could use the [client-go][client_go] API for example to -check Kube resources within your tests, or even create custom resources. Your -custom test image is being executed within a Pod, so you can use an in-cluster +In golang, you could use the [client-go][client_go] API for example to +check Kube resources within your tests, or even create custom resources. Your +custom test image is being executed within a Pod, so you can use an in-cluster connection to invoke the Kube API. @@ -246,3 +300,4 @@ connection to invoke the Kube API. [user_doc]: /docs/advanced-topics/scorecard/scorecard/ [scorecard_binary]: https://github.com/operator-framework/operator-sdk/blob/master/internal/scorecard/examples/custom-scorecard-tests/images/custom-scorecard-tests/cmd/test/main.go [sample_makefile]: https://github.com/operator-framework/operator-sdk/blob/master/internal/scorecard/examples/custom-scorecard-tests/Makefile +[kustomize-patchJson6902]: https://kubernetes-sigs.github.io/kustomize/api-reference/kustomization/patchesjson6902/ diff --git a/website/content/en/docs/advanced-topics/scorecard/scorecard.md b/website/content/en/docs/advanced-topics/scorecard/scorecard.md index 532d5bbda9..1f10db9699 100644 --- a/website/content/en/docs/advanced-topics/scorecard/scorecard.md +++ b/website/content/en/docs/advanced-topics/scorecard/scorecard.md @@ -30,24 +30,28 @@ require if the tests are designed for resource creation. ## Running the Scorecard -1. Generate [bundle][quickstart-bundle] manifests and metadata for your Operator. +1. A default set of kustomize files should have been scaffolded by `operator-sdk init`. +If that is not the case, run `operator-sdk init` as you would have to initialize your project +and copy scaffolded files: + ```sh + $ TMP_PROJECT="$(mktemp -d)/" + $ mkdir "$TMP_PROJECT" + $ pushd "$TMP_PROJECT" + $ operator-sdk init + $ popd + $ cp -r "$TMP_PROJECT"/config/scorecard ./config/ + ``` +The default config generated by this kustomization can be immediately run against your operator. +See the [config file section](#config-file) for an explanation of the configuration file format. +1. (Re)enerate [bundle][quickstart-bundle] manifests and metadata for your Operator. `make bundle` will automatically add scorecard annotations to your bundle's metadata, which is used by the `scorecard` command to run tests. -1. Define a scorecard configuration file `bundle/tests/scorecard/config.yaml`, the default path. -If you choose to define this file elsewhere, you can either change the modified portion of that path in -`annotations.yaml` and `bundle.Dockerfile`, or override that value using the `--config` flag. -Unless executing custom tests, you may copy this [sample][sample-config] configuration file into your project. -See the [config file section](#config-file) for an explanation of the configuration file format. -1. Add the following line to the end of your `bundle.Dockerfile`: - ```docker - COPY bundle/tests/scorecard /tests/scorecard - ``` 1. Execute the [`scorecard` command][cli-scorecard]. See the [command args section](#command-args) for an overview of command invocation. ## Configuration -The scorecard test execution is driven by a configuration file named `config.yaml`. +The scorecard test execution is driven by a configuration file named `config.yaml`, generated by `make bundle`. The configuration file is located at the following location within your bundle directory (`bundle/` by default): ```sh $ tree ./bundle @@ -60,23 +64,24 @@ $ tree ./bundle ### Config File -A complete scorecard configuration file can be found [here][sample-config] -and used for running the scorecard pre-defined tests that ship with the SDK. - -A sample of the scorecard configuration file may look as follows: +The following YAML spec is an example of the scorecard configuration file: ```yaml +kind: Configuration +apiversion: scorecard.operatorframework.io/v1alpha3 +metadata: + name: config stages: - parallel: true tests: - - image: quay.io/operator-framework/scorecard-test:dev + - image: quay.io/operator-framework/scorecard-test:latest entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - - image: quay.io/operator-framework/scorecard-test:dev + - image: quay.io/operator-framework/scorecard-test:latest entrypoint: - scorecard-test - olm-bundle-validation @@ -190,7 +195,7 @@ See an example of the JSON format produced by a scorecard test: "kind": "Test", "apiVersion": "scorecard.operatorframework.io/v1alpha3", "spec": { - "image": "quay.io/operator-framework/scorecard-test:dev", + "image": "quay.io/operator-framework/scorecard-test:latest", "entrypoint": [ "scorecard-test", "olm-bundle-validation" @@ -220,7 +225,7 @@ See an example of the text format produced by a scorecard test: ``` -------------------------------------------------------------------------------- -Image: quay.io/operator-framework/scorecard-test:dev +Image: quay.io/operator-framework/scorecard-test:latest Entrypoint: [scorecard-test olm-bundle-validation] Labels: "suite":"olm" @@ -254,7 +259,7 @@ Scorecard will execute custom tests if they follow these mandated conventions: * tests can obtain the bundle contents at a shared mount point of /bundle * tests can access the Kubernetes API using an in-cluster client connection -See [here][custom-image] for an example of a custom test image written in golang. +See [here][custom-image] for an example of a custom test image written in Go. Writing custom tests in other programming languages is possible if the test image follows the above guidelines.