From b136050297046f1e41524f56872361cb3504a226 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 9 Mar 2022 14:19:52 +0100 Subject: [PATCH 1/6] Include Test storage backend in Capact installation for testing purposes --- deploy/kubernetes/charts/capact/Chart.yaml | 4 +++ .../charts/test-storage-backend/.helmignore | 23 ++++++++++++++++++ .../charts/test-storage-backend/Chart.lock | 6 +++++ .../charts/test-storage-backend/Chart.yaml | 13 ++++++++++ .../charts/secret-storage-backend-0.6.0.tgz | Bin 0 -> 2636 bytes .../charts/test-storage-backend/values.yaml | 4 +++ deploy/kubernetes/charts/capact/values.yaml | 3 +++ hack/lib/utilities.sh | 8 ++++-- 8 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 deploy/kubernetes/charts/capact/charts/test-storage-backend/.helmignore create mode 100644 deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.lock create mode 100644 deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml create mode 100644 deploy/kubernetes/charts/capact/charts/test-storage-backend/charts/secret-storage-backend-0.6.0.tgz create mode 100644 deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml diff --git a/deploy/kubernetes/charts/capact/Chart.yaml b/deploy/kubernetes/charts/capact/Chart.yaml index 20a8c3704..55b4def74 100644 --- a/deploy/kubernetes/charts/capact/Chart.yaml +++ b/deploy/kubernetes/charts/capact/Chart.yaml @@ -30,3 +30,7 @@ dependencies: - name: dashboard version: "0.0.1" repository: "file://charts/dashboard" +- name: test-storage-backend + version: "0.0.1" + repository: "file://charts/test-storage-backend" + condition: testStorageBackend.enabled diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/.helmignore b/deploy/kubernetes/charts/capact/charts/test-storage-backend/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.lock b/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.lock new file mode 100644 index 000000000..80acff72b --- /dev/null +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: secret-storage-backend + repository: file://../../../secret-storage-backend + version: 0.6.0 +digest: sha256:e4c71dc2ae9e79dc279b9c151426ed7c018a15d698465e603a41b4c6de56a198 +generated: "2022-03-09T13:27:29.340049+01:00" diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml b/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml new file mode 100644 index 000000000..b176765be --- /dev/null +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: test-storage-backend +description: A wrapper for Secret Storage Backend Helm chart for testing purporses + +type: application + +version: 0.1.0 +appVersion: 0.0.1 + +dependencies: + - name: secret-storage-backend + version: "^0.6.0" + repository: "file://../../../secret-storage-backend" diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/charts/secret-storage-backend-0.6.0.tgz b/deploy/kubernetes/charts/capact/charts/test-storage-backend/charts/secret-storage-backend-0.6.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..88328b42d65ae485f528a89dc10ab66fa90f5b64 GIT binary patch literal 2636 zcmV-S3bXYeiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH$lZ`(Mweb%oy*SQ1C^p0iON!-G~eaKBRZE@4ah|?~17mI+F zj%`jUQbSU1-8B2#7bGQ{lI7T~yP3{x_z)}N;rSxZC#5i!sM|^lNv7ByllVRIq;onW zQiltYvM2kGD2k%~(UJX)qNx5G^^Utwx<|ckuiNb(^$(v!-Q(lv@CiiUfcDmrD^28+ zXm46|;r<|lWb_?MMFk(gycduxTmBn`QP>RTr(j~cmg1S(vzN*#)ovkEN2fzyWc9pk^Xx35*e9 ziA19j!95K?%N#AD6V8RU{#1b$K$67#80FQ_1ps6PYXvatuV-l024d2*${@=Xh8FhK z1h~+kP}_}jyx6^xIsOd9A)*SV$Wa=OAy<@7E#gVen4Q;{W@hV_`_*a%K_-%u8h7U1 z2;*E*y*L$I<43*ZdZJ!Pk!J(wMNwol+ze*U=a@38DJrLTfJBO{d~UW$zAp#J);LOFnL!HU(Nq2MCkDa(~c`D$nmppbe1$5C|L zHK#|3T*i(wMpLSlv#~hKEw#> zD%9E6rUF~y|6#v-RE__=s2d$U#{Y*Hx3`_A!THC`cx+vB_BPEAgQuOlyWsY=4GB&t zM`&$G56sdQv{%43R)?;Xd}Hv2?_64GAj@n3J{xz*W9T2+N1BfE$%KA{R=WiSE>lw6 z-35VV_SD4{vZP=_#{z!K2{R!xF=>GnBMkn64!4EZhJq1E!5CwbD+DD{g#XXSC^>3W zPWTB$mM9<+o!iACWKsn`ju%!jXKzLZ*2FHAVf<4VyqbVShC~$_Eb=2&lu9o0Y4|`H zGZ8yihED~=GGB1JtDehHxzaVAELB@hF4bFrY@FXk{Y)+K{I?Zl@V~q7?~2EzQXG=- z%4y84E%~gx{6gk3Nx7at>j%~TLA7eQ9MRoRTk~V(40@Yu?5C($&BoddE>4SsVq}bL z_f58oRy$mm%vZX5E;3>G3?}a+F+%HqTF|;_eYt^&G!@)2tafJDfjf?8$kI^FI#!4Q z+};A^G0T&!;)af)t>CK26D|s(p?As(eVirW&vs{!bO?Zq|seD6i zvWI#AX|5K&Kw!Tzdb7#pRl&ld?x=;KAGfJv;=}gt?aCKW=NQxeX37d2A)UHd)|aZ zlF)xY5wAgB7^WtiFdUJAic2(oc-dhZyHLx4d+PMDZ`N_rg z%fH==!brCn!^P!`7w1W{zZzku5jl7uW(X79Q%ZRpS`khSiPyl0ly8r zT3l%k@8kd1=#@QXFOHU7=5I>JYi6&Y{0{fmQQJ5F+nWE+GP3VA@Rt0)+pp*U-Dkb% z*<=3y5Touen)ByMm9PTo%zIG7NM3jR$kT9wCw&1xzc~M zi>t@0A|e+~r3}z3N942+XZiAtBXBZD<19QP8wJ^gql)j`D&0Nzh+iq7yBOV9#;tsP zKRq9RZ1w*RV6PCc?fbu@Ucc`DpB;7|{r^LZtqEi01HsbMyGs1K>kDZ~`Sh${sG$a_ zNmpC`9a-hWjUDXYc0TrT0Nd=pU$_5W)Quj$|9gq<1d5d(pT&qh`X< z^g221qI1yzZzjBA%*Juo%<6kKVJL(zkB>>{|_+^U`Vt^>AnwlI}RUa$YGpQmQX&0jKuHB6jc};!1atO zD7w%J)C?JeDHCINJ4*TVXOPH9u!n4uMNDGO<0TkaSy*nL2LZu4ADb*eO*Rdaj<4>|<|CTRi(~kL5 z{;TG^vqEIfe4fFCGE~9SP<_aPr{S2q51xiPHP1rQY4G%)!2!G@l8RixtFvIA literal 0 HcmV?d00001 diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml b/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml new file mode 100644 index 000000000..971efa7cf --- /dev/null +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml @@ -0,0 +1,4 @@ +supportedProviders: + - "dotenv" + # this provider doesn't have AWS credentials configured on purpose + - "aws_secretsmanager" diff --git a/deploy/kubernetes/charts/capact/values.yaml b/deploy/kubernetes/charts/capact/values.yaml index 04787557b..2ad366807 100644 --- a/deploy/kubernetes/charts/capact/values.yaml +++ b/deploy/kubernetes/charts/capact/values.yaml @@ -28,6 +28,9 @@ integrationTest: name: e2e-test pullPolicy: IfNotPresent +testStorageBackend: + enabled: false + postInstallTypeInstanceJob: image: name: populator diff --git a/hack/lib/utilities.sh b/hack/lib/utilities.sh index 665c9275a..f286f43b6 100644 --- a/hack/lib/utilities.sh +++ b/hack/lib/utilities.sh @@ -201,7 +201,7 @@ capact::delete_cluster() { # - DOCKER_REPOSITORY # - PRINT_INSECURE_NOTES # - ENABLE_POPULATOR - if set to true then database populator will be enabled and it will populate database with manifests -# - USE_TEST_SETUP - if set to true, then a test policy is configured +# - USE_TEST_SETUP - if set to true, then a test policy and storage are configured # - INCREASE_RESOURCE_LIMITS - if set to true, then the components will use higher resource requests and limits # - HUB_MANIFESTS_SOURCE_REPO_REF - set this to override the Git branch from which the source manifests are populated # - HUB_MANIFESTS_SOURCE_REPO_URL - set this to override the Git URL from which the source manifests are populated @@ -228,7 +228,7 @@ capact::install() { CAPACT_OVERRIDES+=",global.containerRegistry.overrideTag=${DOCKER_TAG}" CAPACT_OVERRIDES+=",hub-public.populator.enabled=${ENABLE_POPULATOR}" - CAPACT_OVERRIDES+=",engine.testSetup.enabled=${USE_TEST_SETUP}" + CAPACT_OVERRIDES+=",engine.testSetup.enabled=${USE_TEST_SETUP},testStorageBackend.enabled=${USE_TEST_SETUP}" CAPACT_OVERRIDES+=",notes.printInsecure=${PRINT_INSECURE_NOTES}" if [[ "${DISABLE_KUBED_INSTALLATION:-"false"}" == "true" ]]; then @@ -259,6 +259,10 @@ capact::install() { BUILD_IMAGES_LIST="" fi + if [[ "${USE_TEST_SETUP}" == "true" ]]; then + BUILD_IMAGES_LIST=+",secret-storage-backend" + fi + if [ -n "${CAPACT_HELM_REPO:-}" ]; then CAPACT_INSTALL_ADDITIONAL_OPTS="${CAPACT_INSTALL_ADDITIONAL_OPTS} --helm-repo=${CAPACT_HELM_REPO}" fi From e8cf713932e67362adb3bc0c9f5cf0e58061c461 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 9 Mar 2022 15:37:48 +0100 Subject: [PATCH 2/6] Register Test Storage Backend --- cmd/populator/cmd/register/register.go | 1 + cmd/populator/cmd/register/test-storage.go | 28 +++++ .../charts/test-storage-backend/values.yaml | 3 + .../templates/post-install-type-instance.yaml | 14 ++- hack/lib/utilities.sh | 2 +- internal/cli/credstore/config_darwin.go | 4 +- internal/cli/testing/storage_backend.go | 110 ++++++++++++++++++ 7 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 cmd/populator/cmd/register/test-storage.go create mode 100644 internal/cli/testing/storage_backend.go diff --git a/cmd/populator/cmd/register/register.go b/cmd/populator/cmd/register/register.go index 201d2bbb4..059dcc344 100644 --- a/cmd/populator/cmd/register/register.go +++ b/cmd/populator/cmd/register/register.go @@ -13,6 +13,7 @@ func NewRegister(cliName string) *cobra.Command { hub.AddCommand( NewCapactInstallation(cliName), + NewTestStorageBackend(cliName), NewOCFManifests(cliName), ) return hub diff --git a/cmd/populator/cmd/register/test-storage.go b/cmd/populator/cmd/register/test-storage.go new file mode 100644 index 000000000..828bb9064 --- /dev/null +++ b/cmd/populator/cmd/register/test-storage.go @@ -0,0 +1,28 @@ +package register + +import ( + "capact.io/capact/internal/cli/testing" + "github.com/docker/cli/cli" + "github.com/spf13/cobra" + + "capact.io/capact/internal/cli/heredoc" +) + +// NewTestStorageBackend returns a cobra.Command for populating storage backend TypeInstance for testing purposes. +func NewTestStorageBackend(cliName string) *cobra.Command { + return &cobra.Command{ + Use: "test-storage-backend", + Short: "Produces and uploads TypeInstances which describe storage backend for testing purposes", + Example: heredoc.WithCLIName(` + test-storage-backend + `, cliName), + Args: cli.RequiresMaxArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + storageRegister, err := testing.NewStorageBackendRegister() + if err != nil { + return err + } + return storageRegister.RegisterTypeInstances(cmd.Context()) + }, + } +} diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml b/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml index 971efa7cf..33c206d20 100644 --- a/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml @@ -1,3 +1,6 @@ +secret-storage-backend: + fullnameOverride: "capact-test-storage-backend" + supportedProviders: - "dotenv" # this provider doesn't have AWS credentials configured on purpose diff --git a/deploy/kubernetes/charts/capact/templates/post-install-type-instance.yaml b/deploy/kubernetes/charts/capact/templates/post-install-type-instance.yaml index 0d28d7f3e..6a8709cc7 100644 --- a/deploy/kubernetes/charts/capact/templates/post-install-type-instance.yaml +++ b/deploy/kubernetes/charts/capact/templates/post-install-type-instance.yaml @@ -58,7 +58,7 @@ spec: restartPolicy: Never serviceAccountName: post-install-type-instances-job containers: - - name: job + - name: installation image: "{{ .Values.global.containerRegistry.path }}/{{ .Values.postInstallTypeInstanceJob.image.name }}:{{ .Values.global.containerRegistry.overrideTag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.postInstallTypeInstanceJob.image.pullPolicy }} env: @@ -72,3 +72,15 @@ spec: value: "http://capact-hub-local.{{ .Release.Namespace }}/graphql" command: ["/bin/sh", "-c"] args: {{ .Values.postInstallTypeInstanceJob.args }} + {{ if .Values.testStorageBackend.enabled -}} + - name: test-storage + image: "{{ .Values.global.containerRegistry.path }}/{{ .Values.postInstallTypeInstanceJob.image.name }}:{{ .Values.global.containerRegistry.overrideTag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.postInstallTypeInstanceJob.image.pullPolicy }} + env: + - name: TEST_STORAGE_BACKEND_URL + value: "capact-test-storage-backend.{{ .Release.Namespace }}:50051" + - name: LOCAL_HUB_ENDPOINT + value: "http://capact-hub-local.{{ .Release.Namespace }}/graphql" + command: [ "/bin/sh", "-c" ] + args: ["/app register test-storage-backend"] + {{- end }} diff --git a/hack/lib/utilities.sh b/hack/lib/utilities.sh index f286f43b6..19266103d 100644 --- a/hack/lib/utilities.sh +++ b/hack/lib/utilities.sh @@ -260,7 +260,7 @@ capact::install() { fi if [[ "${USE_TEST_SETUP}" == "true" ]]; then - BUILD_IMAGES_LIST=+",secret-storage-backend" + BUILD_IMAGES_LIST+=",secret-storage-backend" fi if [ -n "${CAPACT_HELM_REPO:-}" ]; then diff --git a/internal/cli/credstore/config_darwin.go b/internal/cli/credstore/config_darwin.go index 6d70e26c4..e62f411ac 100644 --- a/internal/cli/credstore/config_darwin.go +++ b/internal/cli/credstore/config_darwin.go @@ -4,8 +4,9 @@ package credstore import ( - "capact.io/capact/internal/cli/config" "fmt" + + "capact.io/capact/internal/cli/config" "github.com/99designs/keyring" "github.com/pkg/errors" ) @@ -32,5 +33,4 @@ func openStore() (Keyring, error) { default: return nil, errors.New("backend not supported") } - } diff --git a/internal/cli/testing/storage_backend.go b/internal/cli/testing/storage_backend.go new file mode 100644 index 000000000..cea21cfbb --- /dev/null +++ b/internal/cli/testing/storage_backend.go @@ -0,0 +1,110 @@ +package testing + +import ( + "context" + "encoding/json" + + "capact.io/capact/internal/logger" + "capact.io/capact/internal/ptr" + hublocalgraphql "capact.io/capact/pkg/hub/api/graphql/local" + "capact.io/capact/pkg/hub/client/local" + "github.com/pkg/errors" + "github.com/vrischmann/envconfig" + "go.uber.org/zap" +) + +// StorageBackendRegisterConfig holds configuration for StorageBackendRegister. +type StorageBackendRegisterConfig struct { + Logger logger.Config + LocalHubEndpoint string `envconfig:"default=http://capact-hub-local.capact-system/graphql"` + TestStorageBackendURL string `envconfig:"default=capact-test-storage-backend.capact-system:50051"` +} + +const testStorageTypePath = "cap.type.capactio.capact.validation.storage" +const testStorageTypeRevision = "0.1.0" +const testStorageTypeContextSchema = ` + { + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "required": [ + "provider" + ], + "properties": { + "provider": { + "$id": "#/properties/context/properties/provider", + "type": "string", + "enum": [ + "aws_secretsmanager", + "dotenv" + ] + } + }, + "additionalProperties": false + } +` + +type typeInstanceValue struct { + URL string `json:"url"` + AcceptValue bool `json:"acceptValue"` + ContextSchema interface{} `json:"contextSchema"` +} + +// StorageBackendRegister provides functionality to produce and upload test storage backend TypeInstance. +type StorageBackendRegister struct { + logger *zap.Logger + localHubCli *local.Client + cfg StorageBackendRegisterConfig +} + +// NewStorageBackendRegister returns a new StorageBackendRegister instance. +func NewStorageBackendRegister() (*StorageBackendRegister, error) { + var cfg StorageBackendRegisterConfig + err := envconfig.Init(&cfg) + if err != nil { + return nil, errors.Wrap(err, "while loading configuration") + } + + logger, err := logger.New(cfg.Logger) + if err != nil { + return nil, errors.Wrap(err, "while creating zap logger") + } + + client := local.NewDefaultClient(cfg.LocalHubEndpoint) + + return &StorageBackendRegister{ + logger: logger, + localHubCli: client, + cfg: cfg, + }, nil +} + +// RegisterTypeInstances produces and uploads TypeInstances which describe Test storage backend. +func (i *StorageBackendRegister) RegisterTypeInstances(ctx context.Context) error { + var contextSchema interface{} + err := json.Unmarshal([]byte(testStorageTypeContextSchema), &contextSchema) + if err != nil { + return errors.Wrap(err, "while unmarshaling contextSchema") + } + + in := &hublocalgraphql.CreateTypeInstanceInput{ + CreatedBy: ptr.String("populator/test-storage-backend-registration"), + TypeRef: &hublocalgraphql.TypeInstanceTypeReferenceInput{ + Path: testStorageTypePath, + Revision: testStorageTypeRevision, + }, + Value: typeInstanceValue{ + URL: i.cfg.TestStorageBackendURL, + AcceptValue: true, + ContextSchema: contextSchema, + }, + } + + id, err := i.localHubCli.CreateTypeInstance(ctx, in) + if err != nil { + return errors.Wrap(err, "while creating TypeInstance") + } + + i.logger.Info("Successfully created Test Storage Backend TypeInstance", zap.String("id", id)) + + return nil +} From 1d1aa40d71ecc4d57b2a0e7f2fabc2ab81ba9d87 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 9 Mar 2022 15:48:06 +0100 Subject: [PATCH 3/6] Build image and update test instruction --- cmd/cli/cmd/install.go | 2 +- internal/cli/capact/build.go | 33 ++++++++++++++----- .../api/grpc/storage_backend/client_test.go | 12 ++++++- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/cmd/cli/cmd/install.go b/cmd/cli/cmd/install.go index 02563f074..e5aac9a55 100644 --- a/cmd/cli/cmd/install.go +++ b/cmd/cli/cmd/install.go @@ -55,7 +55,7 @@ func NewInstall() *cobra.Command { flags.StringVar(&opts.Namespace, "namespace", capact.Namespace, "Capact namespace.") flags.StringVar(&opts.Environment, "environment", capact.KindEnv, "Capact environment.") flags.StringSliceVar(&opts.InstallComponents, "install-component", capact.Components.All(), "Components names that should be installed. Takes comma-separated list.") - flags.StringSliceVar(&opts.BuildImages, "build-image", capact.Images.All(), "Local images names that should be build when using @local version. Takes comma-separated list.") + flags.StringSliceVar(&opts.BuildImages, "build-image", capact.Images.DefaultList(), "Local images names that should be build when using @local version. Takes comma-separated list.") flags.BoolVar(&opts.Parameters.IncreaseResourceLimits, "increase-resource-limits", true, "Enables higher resource requests and limits for components.") flags.DurationVar(&opts.Timeout, "timeout", 10*time.Minute, `Maximum time during which the upgrade process is being watched, where "0" means "infinite". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".`) flags.BoolVar(&opts.UpdateHostsFile, "update-hosts-file", true, "Updates /etc/hosts with entry for Capact GraphQL Gateway.") diff --git a/internal/cli/capact/build.go b/internal/cli/capact/build.go index 84dcd6ce7..870f1343e 100644 --- a/internal/cli/capact/build.go +++ b/internal/cli/capact/build.go @@ -16,8 +16,9 @@ import ( ) type image struct { - Dir string - Target string + Dir string + Target string + IgnoredByDefault bool ExtraBuildArgs []string DisableBuildKit bool @@ -25,10 +26,22 @@ type image struct { type images map[string]image -func (i images) All() []string { +func (i images) FullList() []string { + return i.toStringSlice(false) +} + +func (i images) DefaultList() []string { + return i.toStringSlice(true) +} + +func (i images) toStringSlice(skipIgnoredByDefault bool) []string { var all []string - for img := range i { - all = append(all, img) + for key, val := range i { + if skipIgnoredByDefault && val.IgnoredByDefault { + continue + } + + all = append(all, key) } // We generate doc automatically, so it needs to be deterministic @@ -47,8 +60,7 @@ var Images = images{ Target: "generic", }, "hub-js": { - Dir: "hub-js", - + Dir: "hub-js", DisableBuildKit: true, }, "argo-runner": { @@ -71,6 +83,11 @@ var Images = images{ "SOURCE_PATH=./test/e2e/*_test.go", }, }, + "secret-storage-backend": { + Dir: ".", + Target: "generic", + IgnoredByDefault: true, // used only for development and testing purposes + }, } func buildImage(ctx context.Context, status printer.Status, imgName string, img image, repository, version string) (string, error) { @@ -111,7 +128,7 @@ func buildImage(ctx context.Context, status printer.Status, imgName string, img func BuildImages(ctx context.Context, status printer.Status, repository, version string, names []string) ([]string, error) { var created []string - for _, image := range Images.All() { + for _, image := range Images.FullList() { if !slices.Contains(names, image) { continue } diff --git a/pkg/hub/api/grpc/storage_backend/client_test.go b/pkg/hub/api/grpc/storage_backend/client_test.go index 198a38a45..94eff982c 100644 --- a/pkg/hub/api/grpc/storage_backend/client_test.go +++ b/pkg/hub/api/grpc/storage_backend/client_test.go @@ -20,6 +20,12 @@ const typeInstanceIDEnv = "TYPEINSTANCE_ID" // // NOTE: Before running this test, make sure that the server is running under the `srvAddr` address // and the `provider` is enabled on this server. +// +// To run this test, execute: +// GRPC_SECRET_STORAGE_BACKEND_ADDR=":50051" go test ./pkg/hub/api/grpc/storage_backend -run "^TestNewStorageBackendClient$" -v -count 1 +// +// You can run this test with custom TypeInstance ID, by setting TYPEINSTANCE_ID env variable during test run. +// This might be helpful while running this test against server with different default provider configured. func TestNewStorageBackendClient(t *testing.T) { srvAddr := os.Getenv(secretStorageBackendAddrEnv) if srvAddr == "" { @@ -27,7 +33,11 @@ func TestNewStorageBackendClient(t *testing.T) { } value := []byte(`{"key": true}`) - typeInstanceID := "id" + typeInstanceID := os.Getenv(typeInstanceIDEnv) + if typeInstanceID == "" { + // fallback to default + typeInstanceID = "id" + } provider := "dotenv" reqContext := []byte(fmt.Sprintf(`{"provider":"%s"}`, provider)) From 9e7d712b3ceb02ff5ee63abfe2647b2ffbfdbb04 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 9 Mar 2022 16:20:47 +0100 Subject: [PATCH 4/6] Generate docs for Populator --- Makefile | 7 +- cmd/cli/cmd/docs.go | 20 +---- cmd/populator/cmd/docs.go | 27 +++++++ cmd/populator/cmd/root.go | 1 + cmd/populator/docs/populator.md | 22 ++++++ .../populator_register-capact-installation.md | 21 ----- .../docs/populator_register-ocf-manifests.md | 76 ------------------- cmd/populator/docs/populator_register.md | 21 +++++ .../populator_register_capact-installation.md | 29 +++++++ .../docs/populator_register_ocf-manifests.md | 30 ++++++++ ...populator_register_test-storage-backend.md | 29 +++++++ internal/frontmatter/format.go | 24 ++++++ 12 files changed, 191 insertions(+), 116 deletions(-) create mode 100644 cmd/populator/cmd/docs.go create mode 100644 cmd/populator/docs/populator.md delete mode 100644 cmd/populator/docs/populator_register-capact-installation.md delete mode 100644 cmd/populator/docs/populator_register-ocf-manifests.md create mode 100644 cmd/populator/docs/populator_register.md create mode 100644 cmd/populator/docs/populator_register_capact-installation.md create mode 100644 cmd/populator/docs/populator_register_ocf-manifests.md create mode 100644 cmd/populator/docs/populator_register_test-storage-backend.md create mode 100644 internal/frontmatter/format.go diff --git a/Makefile b/Makefile index 7f3d3d340..75dd68d1e 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,7 @@ gen-go-source-code: go generate -x ./... .PHONY: gen-go-source-code -gen-docs: gen-docs-cli ## Generate all documentation +gen-docs: gen-docs-cli gen-docs-populator ## Generate all documentation .PHONY: gen-docs gen-docs-cli: @@ -179,6 +179,11 @@ gen-docs-cli: go run cmd/cli/main.go gen-usage-docs .PHONY: gen-docs-cli +gen-docs-populator: + rm -f ./cmd/populator/docs/* + go run cmd/populator/main.go gen-usage-docs +.PHONY: gen-docs-populator + ############### # Development # ############### diff --git a/cmd/cli/cmd/docs.go b/cmd/cli/cmd/docs.go index 3a2e4318b..8854e65d3 100644 --- a/cmd/cli/cmd/docs.go +++ b/cmd/cli/cmd/docs.go @@ -1,22 +1,13 @@ package cmd import ( - "fmt" - "path/filepath" - "strings" - + "capact.io/capact/internal/frontmatter" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" ) const ( docsTargetDir = "./cmd/cli/docs" - - frontmatterFormat = `--- -title: %s ---- - -` ) // NewDocs returns a cobra.Command for generating Capact CLI documentation. @@ -30,14 +21,7 @@ func NewDocs() *cobra.Command { root.DisableAutoGenTag = true defaultLinkHandler := func(s string) string { return s } - return doc.GenMarkdownTreeCustom(root, docsTargetDir, frontmatterFilePrepender, defaultLinkHandler) + return doc.GenMarkdownTreeCustom(root, docsTargetDir, frontmatter.FilePrepender, defaultLinkHandler) }, } } - -func frontmatterFilePrepender(filePath string) string { - fileName := filepath.Base(filePath) - fileNameWithoutExt := strings.TrimSuffix(fileName, filepath.Ext(fileName)) - title := strings.Replace(fileNameWithoutExt, "_", " ", -1) - return fmt.Sprintf(frontmatterFormat, title) -} diff --git a/cmd/populator/cmd/docs.go b/cmd/populator/cmd/docs.go new file mode 100644 index 000000000..ff5fa6e57 --- /dev/null +++ b/cmd/populator/cmd/docs.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "capact.io/capact/internal/frontmatter" + "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" +) + +const ( + docsTargetDir = "./cmd/populator/docs" +) + +// NewDocs returns a cobra.Command for generating Populator documentation. +func NewDocs(cliName string) *cobra.Command { + return &cobra.Command{ + Use: "gen-usage-docs", + Hidden: true, + Short: "Generate usage documentation", + RunE: func(cmd *cobra.Command, args []string) error { + root := NewRoot(cliName) + root.DisableAutoGenTag = true + + defaultLinkHandler := func(s string) string { return s } + return doc.GenMarkdownTreeCustom(root, docsTargetDir, frontmatter.FilePrepender, defaultLinkHandler) + }, + } +} diff --git a/cmd/populator/cmd/root.go b/cmd/populator/cmd/root.go index ec47b9402..1848efc36 100644 --- a/cmd/populator/cmd/root.go +++ b/cmd/populator/cmd/root.go @@ -22,6 +22,7 @@ func NewRoot(cliName string) *cobra.Command { rootCmd.AddCommand( register.NewRegister(cliName), + NewDocs(cliName), ) return rootCmd diff --git a/cmd/populator/docs/populator.md b/cmd/populator/docs/populator.md new file mode 100644 index 000000000..08a54c6fc --- /dev/null +++ b/cmd/populator/docs/populator.md @@ -0,0 +1,22 @@ +--- +title: populator +--- + +## populator + + + +``` +populator [flags] +``` + +### Options + +``` + -h, --help help for populator +``` + +### SEE ALSO + +* [populator register](populator_register.md) - This command consists of multiple subcommands which allows you to register Capact resources + diff --git a/cmd/populator/docs/populator_register-capact-installation.md b/cmd/populator/docs/populator_register-capact-installation.md deleted file mode 100644 index 646bc3a98..000000000 --- a/cmd/populator/docs/populator_register-capact-installation.md +++ /dev/null @@ -1,21 +0,0 @@ -# populator register capact-installation - -Produces and uploads TypeInstances which describe Capact installation. - -## Usage - -```shell -populator register capact-installation -``` - -## Configuration - -You can set the following environment variables to configure: - -| Name | Required | Default | Description | -|-------------------------|----------|-------------------------------------------------|--------------------------------------------------------------------------------------------------------| -| LOCAL_HUB_ENDPOINT | no | `http://capact-hub-local.capact-system/graphql` | Defines local Hub Endpoint. | -| CAPACT_RELEASE_NAME | no | `capact` | Defines Capact Helm release name. | -| HELM_REPOSITORY_PATH | no | `capact` | Defines Helm chart repository URL where the Capact charts are located. | -| HELM_RELEASES_NS_LOOKUP | yes | - | Defines Kubernetes Namespaces in which Capact components were deployed. It is a comma separated list. | -| LOGGER_DEV_MODE | no | `false` | Enable development mode logging. | diff --git a/cmd/populator/docs/populator_register-ocf-manifests.md b/cmd/populator/docs/populator_register-ocf-manifests.md deleted file mode 100644 index f086f36e4..000000000 --- a/cmd/populator/docs/populator_register-ocf-manifests.md +++ /dev/null @@ -1,76 +0,0 @@ -# populator register ocf-manifests - -Populates the OCF manifests into Neo4j database. It reads manifest from remote or local path, converts them into JSON and uploads to database. - -## Prerequisites - -- [Go](https://golang.org) -- Running Kubernetes cluster with Capact installed - -## Build - -To build the binary install required [Prerequisites](https://capact.io/community/development/development-guide/#prerequisites) and run: - -```shell -make build-tool-populator -``` - -It creates a binary for your platform in the `bin` directory. For example, for Linux systems, it is `bin/populator_linux_amd64/populator`. - -## Usage - -> **CAUTION:** In order to run DB populator manually, make sure the populator inside development cluster is disabled. -> To disable it, run `ENABLE_POPULATOR=false make dev-cluster-update` - -It requires one argument, which is a path to directory with Hub manifests. Internally it uses [go-getter](https://github.com/hashicorp/go-getter) so it can download manifests from different locations and in different formats. Supported protocols in DB populator are Git and local files. - -To be able to use it locally when Capact is running in a Kubernetes cluster, two ports need to -be forwarded: - -```shell -kubectl -n capact-system port-forward svc/neo4j-neo4j 7687:7687 -kubectl -n capact-system port-forward svc/neo4j-neo4j 7474:7474 -``` - -To run it and use manifests, for example from the [`hub-manifests`](https://github.com/capactio/hub-manifests) repo, run: - -```shell -./bin/populator_linux_amd64/populator register ocf-manifests --source {PATH_TO_THE_MAIN_DIRECTORY_OF_THE_REPO} -``` - -To use manifests from private git repo, private key, encoded in base64 format, is needed. -For example command to download manifests from Capact repo would look like this: -```shell -export SSHKEY=`base64 -w0 ~/.ssh/id_rsa` -./populator register ocf-manifests --source git@github.com:capactio/capact.git?sshkey=$SSHKEY -``` - -For better performance populator starts HTTP server to serve manifests converted to JSON files. -Neo4j needs access to this JSON files. `APP_JSON_PUBLISH_ADDR` environment variable should be set -so populator can send a correct link to a Neo4j: - -```shell -APP_JSON_PUBLISH_ADDR=http://{HOST_IP} ./populator register ocf-manifests --source . -``` -Replace `HOST_IP` with your computer IP - -There is an option to run populator with multiple sources with the usage of the `source` flag. -Example, which demonstrates this feature: - ```shell -./populator register ocf-manifests --source {PATH_TO_THE_MAIN_DIRECTORY_OF_THE_SOURCE_1} --source {PATH_TO_THE_MAIN_DIRECTORY_OF_THE_SOURCE_2} -``` - -## Configuration - -You can set the following environment variables to configure the Hub database populator: - -| Name | Required | Default | Description | -| ----------------------------------- | -------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| APP_NEO4J_ADDR | no | `neo4j://localhost:7687` | Neo4j address | -| APP_NEO4J_USER | no | `neo4j` | Neo4j admin user | -| APP_NEO4J_PASSWORD | yes | | Neo4h admin password | -| APP_JSON_PUBLISH_ADDR | yes | | Address on which populator will serve JSON files | -| APP_JSON_PUBLISH_PORT | no | `8080` | Port number on which populator will be listening | -| APP_MANIFESTS_PATH | no | ` ` | Path to a directory in a repository where manifests are stored. In case of many sources the same path is used. | -| APP_UPDATE_ON_GIT_COMMIT | no | `false` | Flag to make populator populate data only when there are new changes in a repository | -| APP_LOGGER_DEV_MODE | no | `false` | Enable development mode logging | diff --git a/cmd/populator/docs/populator_register.md b/cmd/populator/docs/populator_register.md new file mode 100644 index 000000000..3633d69b6 --- /dev/null +++ b/cmd/populator/docs/populator_register.md @@ -0,0 +1,21 @@ +--- +title: populator register +--- + +## populator register + +This command consists of multiple subcommands which allows you to register Capact resources + +### Options + +``` + -h, --help help for register +``` + +### SEE ALSO + +* [populator](populator.md) - +* [populator register capact-installation](populator_register_capact-installation.md) - Produces and uploads TypeInstances which describe Capact installation +* [populator register ocf-manifests](populator_register_ocf-manifests.md) - Populates locally available manifests into Neo4j database +* [populator register test-storage-backend](populator_register_test-storage-backend.md) - Produces and uploads TypeInstances which describe storage backend for testing purposes + diff --git a/cmd/populator/docs/populator_register_capact-installation.md b/cmd/populator/docs/populator_register_capact-installation.md new file mode 100644 index 000000000..ad7b103a2 --- /dev/null +++ b/cmd/populator/docs/populator_register_capact-installation.md @@ -0,0 +1,29 @@ +--- +title: populator register capact-installation +--- + +## populator register capact-installation + +Produces and uploads TypeInstances which describe Capact installation + +``` +populator register capact-installation [flags] +``` + +### Examples + +``` +populator capact-installation + +``` + +### Options + +``` + -h, --help help for capact-installation +``` + +### SEE ALSO + +* [populator register](populator_register.md) - This command consists of multiple subcommands which allows you to register Capact resources + diff --git a/cmd/populator/docs/populator_register_ocf-manifests.md b/cmd/populator/docs/populator_register_ocf-manifests.md new file mode 100644 index 000000000..e320402cb --- /dev/null +++ b/cmd/populator/docs/populator_register_ocf-manifests.md @@ -0,0 +1,30 @@ +--- +title: populator register ocf-manifests +--- + +## populator register ocf-manifests + +Populates locally available manifests into Neo4j database + +``` +populator register ocf-manifests [MANIFEST_PATH] [flags] +``` + +### Examples + +``` +APP_JSON_PUBLISH_ADDR=http://{HOST_IP} populator . + +``` + +### Options + +``` + -h, --help help for ocf-manifests + --source strings Manifests location +``` + +### SEE ALSO + +* [populator register](populator_register.md) - This command consists of multiple subcommands which allows you to register Capact resources + diff --git a/cmd/populator/docs/populator_register_test-storage-backend.md b/cmd/populator/docs/populator_register_test-storage-backend.md new file mode 100644 index 000000000..4914e0b81 --- /dev/null +++ b/cmd/populator/docs/populator_register_test-storage-backend.md @@ -0,0 +1,29 @@ +--- +title: populator register test-storage-backend +--- + +## populator register test-storage-backend + +Produces and uploads TypeInstances which describe storage backend for testing purposes + +``` +populator register test-storage-backend [flags] +``` + +### Examples + +``` +populator test-storage-backend + +``` + +### Options + +``` + -h, --help help for test-storage-backend +``` + +### SEE ALSO + +* [populator register](populator_register.md) - This command consists of multiple subcommands which allows you to register Capact resources + diff --git a/internal/frontmatter/format.go b/internal/frontmatter/format.go new file mode 100644 index 000000000..7e2d9ad36 --- /dev/null +++ b/internal/frontmatter/format.go @@ -0,0 +1,24 @@ +package frontmatter + +import ( + "fmt" + "path/filepath" + "strings" +) + +const ( + frontmatterFormat = `--- +title: %s +--- + +` +) + +// FilePrepender is a function which is used to have custom formatting while generating +// markdown documentation for CLI tools. +func FilePrepender(filePath string) string { + fileName := filepath.Base(filePath) + fileNameWithoutExt := strings.TrimSuffix(fileName, filepath.Ext(fileName)) + title := strings.Replace(fileNameWithoutExt, "_", " ", -1) + return fmt.Sprintf(frontmatterFormat, title) +} From c9a79a9af9e4fe23f3f9ba024c7ad7b07ec05e83 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Wed, 9 Mar 2022 16:28:25 +0100 Subject: [PATCH 5/6] Fix providers --- .../charts/capact/charts/test-storage-backend/values.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml b/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml index 33c206d20..c35efd390 100644 --- a/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/values.yaml @@ -1,7 +1,5 @@ secret-storage-backend: fullnameOverride: "capact-test-storage-backend" - -supportedProviders: - - "dotenv" - # this provider doesn't have AWS credentials configured on purpose - - "aws_secretsmanager" + supportedProviders: + - "dotenv" + - "aws_secretsmanager" # this provider doesn't have AWS credentials configured - but it is on purpose From 9bc8e84288bde35a218f567fb0fc6a49ec6f86fb Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Thu, 10 Mar 2022 16:19:04 +0100 Subject: [PATCH 6/6] Improve comment and chart description --- .../charts/capact/charts/test-storage-backend/Chart.yaml | 2 +- internal/frontmatter/format.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml b/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml index b176765be..b632f3468 100644 --- a/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml +++ b/deploy/kubernetes/charts/capact/charts/test-storage-backend/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: test-storage-backend -description: A wrapper for Secret Storage Backend Helm chart for testing purporses +description: A wrapper for Secret Storage Backend Helm chart for testing purposes type: application diff --git a/internal/frontmatter/format.go b/internal/frontmatter/format.go index 7e2d9ad36..2b4cf866f 100644 --- a/internal/frontmatter/format.go +++ b/internal/frontmatter/format.go @@ -14,8 +14,7 @@ title: %s ` ) -// FilePrepender is a function which is used to have custom formatting while generating -// markdown documentation for CLI tools. +// FilePrepender is a function which is used for custom formatting while generating markdown documentation. func FilePrepender(filePath string) string { fileName := filepath.Base(filePath) fileNameWithoutExt := strings.TrimSuffix(fileName, filepath.Ext(fileName))