From 9397f3bab610a91c37aaf653b821d751d7f4537e Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Mon, 13 Jul 2020 13:47:39 -0400 Subject: [PATCH 1/2] internal/scorecard/alpha: add parallelism support --- changelog/fragments/scorecard-parallel.yaml | 55 +++++++++ internal/registry/labels.go | 2 +- internal/scorecard/alpha/config.go | 7 +- .../bundle/tests/scorecard/config.yaml | 31 ++--- internal/scorecard/alpha/formatting.go | 20 ++-- internal/scorecard/alpha/labels_test.go | 99 ++++++++-------- internal/scorecard/alpha/run_test.go | 106 +++++++++++++++++- internal/scorecard/alpha/scorecard.go | 77 +++++++++---- .../scorecard/alpha/testdata/bundle.tar.gz | Bin 2784 -> 2804 bytes .../bundle/tests/scorecard/config.yaml | 88 ++++++++------- .../content/en/docs/scorecard/custom-tests.md | 19 ++-- .../content/en/docs/scorecard/kuttl-tests.md | 11 +- .../en/docs/scorecard/scorecard-alpha.md | 79 ++++++++----- 13 files changed, 408 insertions(+), 186 deletions(-) create mode 100644 changelog/fragments/scorecard-parallel.yaml diff --git a/changelog/fragments/scorecard-parallel.yaml b/changelog/fragments/scorecard-parallel.yaml new file mode 100644 index 0000000000..862e3dd4cb --- /dev/null +++ b/changelog/fragments/scorecard-parallel.yaml @@ -0,0 +1,55 @@ +entries: + - description: > + Changes the scorecard configuration format to include a new top-level + `stages` field, which allows users to define stages of tests and to + enable parallelism within each stage. + + kind: "change" + + # Is this a breaking change? + breaking: true + + # NOTE: ONLY USE `pull_request_override` WHEN ADDING THIS + # FILE FOR A PREVIOUSLY MERGED PULL_REQUEST! + # + # The generator auto-detects the PR number from the commit + # message in which this file was originally added. + # + # What is the pull request number (without the "#")? + # pull_request_override: 0 + + + # Migration can be defined to automatically add a section to + # the migration guide. This is required for breaking changes. + migration: + header: Add stages to scorecard configuration file + body: | + Update your scorecard configuration file to contain a new top-level + stages field, and move your test definitions under one or more stages. + + **Example** + + ```yaml + tests: + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + ``` + + should be converted to + + ```yaml + stages: + - tests: + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + ``` diff --git a/internal/registry/labels.go b/internal/registry/labels.go index ed03778963..b51ec23156 100644 --- a/internal/registry/labels.go +++ b/internal/registry/labels.go @@ -27,7 +27,7 @@ import ( registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle" log "github.com/sirupsen/logrus" - // TODO: replace `gopkg.in/yaml.v2` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the + // TODO: replace `gopkg.in/yaml.v3` with `sigs.k8s.io/yaml` once operator-registry has `json` tags in the // annotations struct. yaml "gopkg.in/yaml.v3" ) diff --git a/internal/scorecard/alpha/config.go b/internal/scorecard/alpha/config.go index f05f8016cd..2eef8a6045 100644 --- a/internal/scorecard/alpha/config.go +++ b/internal/scorecard/alpha/config.go @@ -25,6 +25,11 @@ const ( ConfigDirPath = "/tests/" + ConfigDirName + "/" ) +type Stage struct { + Parallel bool `yaml:"parallel"` + Tests []Test `yaml:"tests"` +} + type Test struct { // Image is the name of the testimage Image string `json:"image"` @@ -37,7 +42,7 @@ type Test struct { // Config represents the set of test configurations which scorecard // would run based on user input type Config struct { - Tests []Test `yaml:"tests"` + Stages []Stage `yaml:"stages"` } // LoadConfig will find and return the scorecard config, the config file diff --git a/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index 8ccedd19e4..103a2ec1b4 100644 --- a/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -1,15 +1,16 @@ -tests: -- image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest1 - labels: - suite: custom - test: customtest1 -- image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest2 - labels: - suite: custom - test: customtest2 +stages: +- tests: + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest1 + labels: + suite: custom + test: customtest1 + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest2 + labels: + suite: custom + test: customtest2 diff --git a/internal/scorecard/alpha/formatting.go b/internal/scorecard/alpha/formatting.go index 15d0fc18f8..389a3698a6 100644 --- a/internal/scorecard/alpha/formatting.go +++ b/internal/scorecard/alpha/formatting.go @@ -42,17 +42,17 @@ func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1 // run based on user selection func (o Scorecard) List() v1alpha3.TestList { output := v1alpha3.NewTestList() - tests := o.selectTests() - - for _, test := range tests { - item := v1alpha3.NewTest() - item.Spec = v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, + for _, stage := range o.Config.Stages { + tests := o.selectTests(stage) + for _, test := range tests { + item := v1alpha3.NewTest() + item.Spec = v1alpha3.TestSpec{ + Image: test.Image, + Entrypoint: test.Entrypoint, + Labels: test.Labels, + } + output.Items = append(output.Items, item) } - output.Items = append(output.Items, item) } - return output } diff --git a/internal/scorecard/alpha/labels_test.go b/internal/scorecard/alpha/labels_test.go index 1959f98418..8bd4d2d605 100644 --- a/internal/scorecard/alpha/labels_test.go +++ b/internal/scorecard/alpha/labels_test.go @@ -55,7 +55,7 @@ func TestEmptySelector(t *testing.T) { return } - tests := o.selectTests() + tests := o.selectTests(o.Config.Stages[0]) testsSelected := len(tests) if testsSelected != c.testsSelected { t.Errorf("Wanted testsSelected %d, got: %d", c.testsSelected, testsSelected) @@ -65,51 +65,52 @@ func TestEmptySelector(t *testing.T) { } } -const testConfig = `tests: -- image: quay.io/someuser/customtest1:v0.0.1 - entrypoint: - - custom-test - labels: - suite: custom - test: customtest1 -- image: quay.io/someuser/customtest2:v0.0.1 - entrypoint: - - custom-test - labels: - suite: custom - test: customtest2 -- image: quay.io/redhat/basictests:v0.0.1 - entrypoint: - - scorecard-test - - basic-check-spec - labels: - suite: basic - test: basic-check-spec-test -- image: quay.io/redhat/basictests:v0.0.1 - entrypoint: - - scorecard-test - - basic-check-status - labels: - suite: basic - test: basic-check-status-test -- image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: - - scorecard-test - - olm-bundle-validation - labels: - suite: olm - test: olm-bundle-validation-test -- image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: - - scorecard-test - - olm-crds-have-validation - labels: - suite: olm - test: olm-crds-have-validation-test -- image: quay.io/redhat/kuttltests:v0.0.1 - labels: - suite: kuttl - entrypoint: - - kuttl-test - - olm-status-descriptors - ` +const testConfig = `stages: +- tests: + - image: quay.io/someuser/customtest1:v0.0.1 + entrypoint: + - custom-test + labels: + suite: custom + test: customtest1 + - image: quay.io/someuser/customtest2:v0.0.1 + entrypoint: + - custom-test + labels: + suite: custom + test: customtest2 + - image: quay.io/redhat/basictests:v0.0.1 + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + - image: quay.io/redhat/basictests:v0.0.1 + entrypoint: + - scorecard-test + - basic-check-status + labels: + suite: basic + test: basic-check-status-test + - image: quay.io/redhat/olmtests:v0.0.1 + entrypoint: + - scorecard-test + - olm-bundle-validation + labels: + suite: olm + test: olm-bundle-validation-test + - image: quay.io/redhat/olmtests:v0.0.1 + entrypoint: + - scorecard-test + - olm-crds-have-validation + labels: + suite: olm + test: olm-crds-have-validation-test + - image: quay.io/redhat/kuttltests:v0.0.1 + labels: + suite: kuttl + entrypoint: + - kuttl-test + - olm-status-descriptors +` diff --git a/internal/scorecard/alpha/run_test.go b/internal/scorecard/alpha/run_test.go index f18eaa4c43..5bd4b1e928 100644 --- a/internal/scorecard/alpha/run_test.go +++ b/internal/scorecard/alpha/run_test.go @@ -16,6 +16,7 @@ package alpha import ( "context" + "errors" "path/filepath" "testing" "time" @@ -25,8 +26,8 @@ import ( "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) +// TODO(joelanford): rewrite to use ginkgo/gomega func TestRunTests(t *testing.T) { - cases := []struct { name string configPathValue string @@ -100,3 +101,106 @@ func TestRunTests(t *testing.T) { } } + +// TODO(joelanford): rewrite to use ginkgo/gomega +func TestRunParallelPass(t *testing.T) { + scorecard := getFakeScorecard(true) + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Millisecond) + defer cancel() + + tests, err := scorecard.Run(ctx) + if err != nil { + t.Fatalf("Expected no error, got error: %v", err) + } + if len(tests.Items) != 2 { + t.Fatalf("Expected 2 tests, got %d", len(tests.Items)) + } + for _, test := range tests.Items { + expectPass(t, test) + } +} + +// TODO(joelanford): rewrite to use ginkgo/gomega +func TestRunSequentialPass(t *testing.T) { + scorecard := getFakeScorecard(false) + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Millisecond) + defer cancel() + + tests, err := scorecard.Run(ctx) + if err != nil { + t.Fatalf("Expected no error, got error: %v", err) + } + if len(tests.Items) != 2 { + t.Fatalf("Expected 2 tests, got %d", len(tests.Items)) + } + for _, test := range tests.Items { + expectPass(t, test) + } +} + +// TODO(joelanford): rewrite to use ginkgo/gomega +func TestRunSequentialFail(t *testing.T) { + scorecard := getFakeScorecard(false) + + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Millisecond) + defer cancel() + + _, err := scorecard.Run(ctx) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Expected deadline exceeded error, got: %v", err) + } +} + +func getFakeScorecard(parallel bool) Scorecard { + return Scorecard{ + Config: Config{ + Stages: []Stage{ + { + Parallel: parallel, + Tests: []Test{ + {}, + {}, + }, + }, + }, + }, + TestRunner: FakeTestRunner{ + Sleep: 5 * time.Millisecond, + TestStatus: &v1alpha3.TestStatus{ + Results: []v1alpha3.TestResult{ + { + State: v1alpha3.PassState, + }, + }, + }, + }, + } +} + +func expectPass(t *testing.T, test v1alpha3.Test) { + if len(test.Status.Results) != 1 { + t.Fatalf("Expected 1 results, got %d", len(test.Status.Results)) + } + for _, r := range test.Status.Results { + if len(r.Errors) > 0 { + t.Fatalf("Expected no errors, got %v", r.Errors) + } + if r.State != v1alpha3.PassState { + t.Fatalf("Expected result state %q, got %q", v1alpha3.PassState, r.State) + } + } +} + +func expectDeadlineExceeded(t *testing.T, test v1alpha3.Test) { + if len(test.Status.Results) != 1 { + t.Fatalf("Expected 1 results, got %d", len(test.Status.Results)) + } + for _, r := range test.Status.Results { + if len(r.Errors) != 1 || r.Errors[0] != context.DeadlineExceeded.Error() { + t.Fatalf("Expected error %q error, got %v", context.DeadlineExceeded, r.Errors) + } + if r.State != v1alpha3.FailState { + t.Fatalf("Expected result state %q, got %q", v1alpha3.FailState, r.State) + } + } +} diff --git a/internal/scorecard/alpha/scorecard.go b/internal/scorecard/alpha/scorecard.go index 03fbb4ff96..623e35bf78 100644 --- a/internal/scorecard/alpha/scorecard.go +++ b/internal/scorecard/alpha/scorecard.go @@ -17,6 +17,7 @@ package alpha import ( "context" "fmt" + "sync" "time" v1 "k8s.io/api/core/v1" @@ -53,6 +54,7 @@ type PodTestRunner struct { } type FakeTestRunner struct { + Sleep time.Duration TestStatus *v1alpha3.TestStatus Error error } @@ -65,25 +67,22 @@ func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { return testOutput, err } - tests := o.selectTests() - if len(tests) == 0 { - return testOutput, nil - } - - for _, test := range tests { - result, err := o.TestRunner.RunTest(ctx, test) - if err != nil { - result = convertErrorToStatus(err, "") + for _, stage := range o.Config.Stages { + tests := o.selectTests(stage) + if len(tests) == 0 { + continue } - out := v1alpha3.NewTest() - out.Spec = v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, + output := make(chan v1alpha3.Test, len(tests)) + if stage.Parallel { + o.runStageParallel(ctx, tests, output) + } else { + o.runStageSequential(ctx, tests, output) + } + close(output) + for o := range output { + testOutput.Items = append(testOutput.Items, o) } - out.Status = *result - testOutput.Items = append(testOutput.Items, out) } if !o.SkipCleanup { @@ -94,14 +93,46 @@ func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { return testOutput, nil } +func (o Scorecard) runStageParallel(ctx context.Context, tests []Test, results chan<- v1alpha3.Test) { + var wg sync.WaitGroup + for _, t := range tests { + wg.Add(1) + go func(test Test) { + results <- o.runTest(ctx, test) + wg.Done() + }(t) + } + wg.Wait() +} + +func (o Scorecard) runStageSequential(ctx context.Context, tests []Test, results chan<- v1alpha3.Test) { + for _, test := range tests { + results <- o.runTest(ctx, test) + } +} + +func (o Scorecard) runTest(ctx context.Context, test Test) v1alpha3.Test { + result, err := o.TestRunner.RunTest(ctx, test) + if err != nil { + result = convertErrorToStatus(err, "") + } + + out := v1alpha3.NewTest() + out.Spec = v1alpha3.TestSpec{ + Image: test.Image, + Entrypoint: test.Entrypoint, + Labels: test.Labels, + } + out.Status = *result + return out +} + // selectTests applies an optionally passed selector expression // against the configured set of tests, returning the selected tests -func (o Scorecard) selectTests() []Test { - +func (o *Scorecard) selectTests(stage Stage) []Test { selected := make([]Test, 0) - - for _, test := range o.Config.Tests { - if o.Selector.String() == "" || o.Selector.Matches(labels.Set(test.Labels)) { + for _, test := range stage.Tests { + if o.Selector == nil || o.Selector.String() == "" || o.Selector.Matches(labels.Set(test.Labels)) { // TODO olm manifests check selected = append(selected, test) } @@ -175,10 +206,10 @@ func (r PodTestRunner) RunTest(ctx context.Context, test Test) (*v1alpha3.TestSt // RunTest executes a single test func (r FakeTestRunner) RunTest(ctx context.Context, test Test) (result *v1alpha3.TestStatus, err error) { select { + case <-time.After(r.Sleep): + return r.TestStatus, r.Error case <-ctx.Done(): return nil, ctx.Err() - default: - return r.TestStatus, r.Error } } diff --git a/internal/scorecard/alpha/testdata/bundle.tar.gz b/internal/scorecard/alpha/testdata/bundle.tar.gz index 5e0933ba70ce42e96b559c4ff79b9d16b68f66b5..6ae010d440d63e146e21278db2b6ead299cddbf1 100644 GIT binary patch delta 2798 zcmVqF76>!cWZ+oz@Fc@Om;7IGZ`x6 z_8zvjri9&qHnQZEWM-JX`R`YfzYPTP#q8|S4%Zp>`xj?$6e0ef z>z~B*CzOf`?gNP_e$vRzH|5@cuUslRD{qFHc&&TNLmw^^rA2g$N+dGCH#n!9YASdu zLQO)ViQfW$;3PsHO6zeqOsl1049w7l6iMvE=75xnGGhSJwcq6gEsjy8|5eAorApgp zF-s(2RTeB-QNBnRk(CE602PQBeYi2p#Uw~^$xLQ@%veyG=tTMO`?#c*FW{A!4e~qA^}2Ffuyk=S8@qdfnFk{V_`0Tr2Sg*3ggLjMj@6WMyYAeg@KC3 z3S48YbHhwJ2127FOqX1xI1E9hi;FqvB~>6XmZ*?xyQvCUFhd9jG5Zw*?ZOnLAp&YC z5*9)r_zI;4iGf&f`njO2KnuGhBN{WUL84g$5?-NZhbR}` z35h^|`CRz0)H+svr?a3sx6wdEQNpQScLKq+q_ac|sX8I9kaZMYI79|Zsxi=s#14t6 zV;Se#8hRA|rBL>Hg;TvYcA>PSd{LmSDejcJF-W6dl1)qbFU4z6Mk=>g(>ur)wJNB~ z+)BHgrb@vZK~X+OnJR8BMPw-=4`V^OwqF8&Mv>Qru98_qwbAr%i3#_hg$tL2bD?2| zki;R;7`kxH;gUqiE{Vcd7rT+La*Rm#3%(j)>H|}073e&)Y}HWVh0_>=ns0=ts5JJg ziAH1f^>DM>AJfS48S9v)WnAA(uWs*P{N~N|-SzvM@mrWqt}fw+>#MhKrr>NYAFxD! zX_O730M}70q$XVZaGP*w8-|D$(ox|(ERdUT2>0KswsAp;1u76B0-0~48Hte2*Qs$D zqoNF99EK3sXkQ6;keC<>_sI-r31y)%L$XlKtwUigGa>GsmFJpIw=&I`&kWashaYA{ z(Li-FJ}F<=WMio#wn24DKVu#~jmXn~_*IFW5Ms>4Izq0`8?)y^MnwB2`t*B^XQ>^T zxU{HA$2g18N_BsZU3sq&rpLv8GqmGshA3vr`1%tqk&Cd2I=~d~#&`dorp9P((QwmX zWJ4LHjXv2XH)xhN`W&0wAZu;(m2C3H2pot+t2~>uY?@^(>JiQW)fjE@oUOrsulk5c zN!ELf)Y`$rR@y+V@-%Se5f(3rSK=(2PL|hI9s{Hn;)9k6W_gm3WPusxYiGA7*av0( zS6Hi)yRV!1w^1?;Ff)e7*MSe$e_prWA2sT~{$Myf)qlr;YW?Ut0{IZDN|4m@VNHUeKboo;LTVTIr|NUY=0}_)NWz;4C@DFoj!8KQ!SUQa&ymabS*X_E0US43Eir0sj zkpP#%9-evrOk$cjepr$ar!g&ExeV*3CA9;QrIWJHJZ!7-~>N2DQw;gPd6$)=M z=7cPc;s(cemD+$O(F~mHgy{ zo7@Odf(h}4@k z`J+8689=_;HZ$e)KjXX0e}1?bzq^`F#+O$c;~?$({2)cNNyNaMA`5SEzWr(Td}7@q zm!(@dSARYiEOIq}kqgvC-%_3bvXkM-`!|-|<8W%$+{w!O$<^)n?)~k9{OmOb12nRQ z#_iSQ?fCNQ!}YuIkDCPTph?2mL@*kxeHgQkWUZcNf=~;rbx(k-8lnM?gTRF1n}x06C`A&hdU@Awx3;lV8&eCvwsDaYvbP=CnNv;}{fx5yJZ~)4 zv9URm1P3L5?--a9iD3h^KwbOJsH!7k69sFSwDnm-eXI@IsRR<&HnKwvl}KO56O05` z*g)qYy@|I?n0dm~wW9NT=J^!Qn%p=Z>G6yRKTIQJV^3JaV zDv4tub*(X%CZ+E`@*-i_GjvM%g5l25bHY@0_`8H@dMNENV`bjb0=UQZM)DU>fE9>GFZY})y`5g$A~Cc!;fU^aWpX?qx% zU`aSfmhQryNZ%<wnL$hBPC{-K|~#&n>6 zJ~Bhv3pY-%#NgghF$NZ0e_fyws?kqJW+=PV#@w9l5!emdF_4FNVqwv)>Q8a*_x`%?mk+)zixAM;Oar`%DADemm+V~mlZ_H0u_}bC& z&kxP>|31eIS0Xi-uORk z#Q&bx@Agjd{}@n>|EoOWPefR|(p}BtNgVsY7nFZ0ZVK0L;pG>-(lJn8)UpWQb4Tvq zPv^oP1rG54SUlsm|7QpIf3G{J$Nye;FgWFZM}dRgpPNL27fL{mDzdKH4e}J z`d)A3oprtL;G%!z{O`GRzES^C&-DL_w?DG@KOFA9|8X&D=)XT2jgB1upG)T()&It? z^`3og;n764-kx}YizF{`o8{TpQC{dm%GWlN6P(}#e?s_Q00030|B+pUZvapL0HouM AhyVZp delta 2778 zcmV<03MKXQ72p+rABzY8000000RQY8U325M(a-u7JMM#L+K>|Unet8bIjN_2K8@w3 z51GsjM6M*nAiw~i?sVz>_YS~sO0wiHT`tu=Sk@9)EcSarC?cHBQE7GddeG@~I-|kB ze0rlnr}~-0nb#Y2dOdH@GvmB&-}BC3@QOSYkSI;$S*P=VtH1}x?R%bYB>((IpwNFi zAi)w{{6wOdp&N+kLxhnx!7k(+PIz5iahRJ2##3^nmu_mqb|Tqa73=oXboWPopQPC3<7@K%JH zghUg+1;9yvgg%tk<8GK%OT`$Np$jRJ*oVylDHUbL0HkZb%L!T>qe}m)j(P>?sMkGZ7gs1{ZV>_bL`cWNTuA$Wwd57Xlk1E^EJch`)0_(f6^#|R z##$GKnRE<OWA=h?O6|!K45DsGYD+bzyDM~{G)KVlY zgh22WN(~YNvEcM`L0N$oc1cDwW?IRCa*dJ`X2h7_cR+Xu5m|#ovj!x*Ld_0QF1!;G zf%3V3@L{QSto(L+L3M7Ufrz4nQ@w5nf@?`v-A z&b2l4DEv#I?28JgdTs1NX-WB_KwDGXDR*O#M!zJRmhxYU*Px74Zm*_ykS}UgP?x!t zb~#Ozf;WPqe2y|z++2#tQbZocf^u!Y1dJkouM1rzvxsV=>E99)?m-I|E(zyC!weyb zL!vQs;hMuGiI80qg|9AlBVpwjk?t3KHNeydrqU|Vd1%?Hp~4HNF$Oi?2vJdK>{kxYiF-^<3zL{R#-og0Io9nyl_c!CWFr8do!VlM1Z{JM8*<3zgiP9*4 z8$T;c#vkB3ejCh4-*PZoVPhf3MDs3rZ|dfd~=Ed>hS3gmk`6jnfzv zWeDRiguq7oO1Oi>#89|TW;jbI3ym3)g=%gc3Tv4Oaqp}=*L=E_X~ukJxE4J8Fe8cv zs-5vk`N9?(OC7Nds#E$I^YCdzp2n|#O6-IXVm?`7yPqaiX!Y1kfQ@k7B{d<}kqqRlDO@omQ zWt2AhWSiWeS=#7xY;uFFwb56y$r~eZAQG+eY|*l5ma(WuI0IB;w8e9_2EXcmBO)bP z?=@0u2M=3m1GUP_z?DZ>yd++Uvurw9T~~PwkXncjS|*s)NkWnZW|*&C-JW0{l>J{} ztxoQ~Zsy-c$uz*s7#`mTKHUF#onCL$=>K|y;qcV|9RsTUU#@rM2124RA5pt-_pAX- zO=6mwL^ie7{n?t*M&>NK5f(XrS!<_!_^*?NpP!59^m`>votF%q`*6O|7#94Te^;Gl zm^0&xR9SzV^LOso76UjpG0jZOmT(F$IIoQRQX^&x%uPb3DmeF{w|kmR^;qE>) zfJ

WnU;-8MZ~*&c4u!ZF=jL%$_KB4RmhBgFl4>BjED}FZ^;bV zM%B4@U1J!gU|+1W0FZ;mC}u=An%`}`x&gLz^@G;m+&kEQfonoJ$_=6p1R{z^{jer~ zv}Yv)$XDBDrkws~e0TZJ4>#j?SJTP(@@iuoq~|_AND*xkF)*jd!dskgf0{j?ShvV! z=~nXU&liG4t|oGSfx0-iR2RSOWO(xajb-;ZoSHRvvhsd%bvwR$fBPUmdyT;WjclWF zdo_7GzP$Qy{cil@CP6!Bk}x(Aj0S5T#_S_mtEZVD)BZPO(P?jFsy|;64M{0NMcoQ@6OL#+gPfNsfAyAaFMbSw;kD@Q%)HDjI#bbKUl0| zW0R8w2PJ<$F)$g4VFR^5UHi_csv}|(1#6h}=(C3USR1rc2_&vj2iwZE>_Pa7hA8t_vw2ICcDY~sc;ycddC~ToYdq%uPv79Zt~ctR z?*D%r19JVhEA7S8Z~AP9J@c2DvD7-03^R;DWp zhzwt4jf3^yA9V*NPk6mSuXob_F`%LUc6vnfvK4^M@xz_@zuOs%O8s|j{vY*wr~H2m zu=@8~PJWvDTSC?@746~*Pm#1rBXP_mKg3lF0J)ZHdqC(z+Vs^%wy?j3&> zW6-L<2~Y{u=%=I79?_8Bn3d9dRYnDioHQ>x<^7`V+zJ*|c-=bjNDK!uRL+vDp2G9C zxkqQaIJ0>?*TvK$F>URz@?!E(j+^t2#jbsu`y_@pW~P+CcDOph3qv#i-zQqXWbc3W zsQSO_^#-Q?>ko&+Q~h@g*sA~Zul0ZUm#75yo&R{n{l8rQqtT#ys{f7w)%$<9%-J#M`lPFp-k%Td1KJ;6fMko^PgV*h}j12Xq;93ecVT5D;e&B0w?*w~byZn2t-*Mm||4$t{7NO&vqQRdFQf0PZb&Z4Tzrm [flags] ``` The scorecard requires a positional argument that holds either the @@ -100,6 +103,24 @@ on-disk path to your operator bundle or the name of a bundle image. For further information about the flags see the [CLI documentation][cli-scorecard]. +## Parallelism + +The configuration file allows operator developers to define separate stages for +their tests. Stages run sequentially in the order they are defined in the +configuration file. A stage contains a list of tests and a configurable +`parallel` setting. + +By default (or when a stage explicitly sets `parallel` to `false`), tests in +a stage are run sequentially in the order they are defined in the configuration +file. Running tests one at a time is helpful to guarantee that no two tests +interact and conflict with each other. + +However, if tests are designed to be fully isolated, they can be parallelized. +To run a set of isolated tests in parallel, include them in the same stage and +set `parallel` to `true`. All tests in a parallel stage are executed +simultaneously, and scorecard waits for all of them to finish before proceding +to the next stage. This can make your tests run much faster. + ## Selecting Tests Tests are selected by setting the `--selector` CLI flag to @@ -225,7 +246,7 @@ Scorecard will execute custom tests if they follow these mandated conventions: * tests accept an entrypoint which include a command and arguments * tests produce v1alpha3 scorecard output in JSON format with no extraneous logging in the test output * tests can obtain the bundle contents at a shared mount point of /bundle - * tests can access the Kube API using an in-cluster client connection + * tests can access the Kubernetes API using an in-cluster client connection