From 6e8092d619650c58a4d38baf1689db36094e399b Mon Sep 17 00:00:00 2001 From: Mateusz Szostok Date: Mon, 14 Mar 2022 17:27:05 +0100 Subject: [PATCH 1/2] Fix resolving backend.context in cypher --- hub-js/graphql/local/schema.graphql | 4 +- pkg/hub/api/graphql/local/schema_gen.go | 6 +- pkg/hub/client/local/client_test.go | 325 ++++++++++++++++++++++++ 3 files changed, 332 insertions(+), 3 deletions(-) create mode 100644 pkg/hub/client/local/client_test.go diff --git a/hub-js/graphql/local/schema.graphql b/hub-js/graphql/local/schema.graphql index 838764257..db6c754db 100644 --- a/hub-js/graphql/local/schema.graphql +++ b/hub-js/graphql/local/schema.graphql @@ -503,7 +503,9 @@ type Mutation { RETURN specBackend ', ' - CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: apoc.convert.toJson(item.typeInstance.backend.context)}) + MATCH (latestRevision)-[:SPECIFIED_BY]->(latestSpec: TypeInstanceResourceVersionSpec) + MATCH (latestSpec)-[:WITH_BACKEND]->(oldSpecBackend:TypeInstanceResourceVersionSpecBackend) + CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: oldSpecBackend.context}) RETURN specBackend ', {spec:spec, latestRevision: latestRevision, item: item}) YIELD value as backendRef diff --git a/pkg/hub/api/graphql/local/schema_gen.go b/pkg/hub/api/graphql/local/schema_gen.go index 15e09761b..b572a979b 100644 --- a/pkg/hub/api/graphql/local/schema_gen.go +++ b/pkg/hub/api/graphql/local/schema_gen.go @@ -1103,7 +1103,9 @@ type Mutation { RETURN specBackend ', ' - CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: apoc.convert.toJson(item.typeInstance.backend.context)}) + MATCH (latestRevision)-[:SPECIFIED_BY]->(latestSpec: TypeInstanceResourceVersionSpec) + MATCH (latestSpec)-[:WITH_BACKEND]->(oldSpecBackend:TypeInstanceResourceVersionSpecBackend) + CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: oldSpecBackend.context}) RETURN specBackend ', {spec:spec, latestRevision: latestRevision, item: item}) YIELD value as backendRef @@ -1673,7 +1675,7 @@ func (ec *executionContext) _Mutation_updateTypeInstances(ctx context.Context, f return ec.resolvers.Mutation().UpdateTypeInstances(rctx, args["in"].([]*UpdateTypeInstancesInput)) } directive1 := func(ctx context.Context) (interface{}, error) { - statement, err := ec.unmarshalOString2áš–string(ctx, "CALL {\n UNWIND $in AS item\n RETURN collect(item.id) as allInputIDs\n}\n\n// Check if all TypeInstances were found\nWITH *\nCALL {\n WITH allInputIDs\n MATCH (ti:TypeInstance)\n WHERE ti.id IN allInputIDs\n WITH collect(ti.id) as foundIDs\n RETURN foundIDs\n}\nCALL apoc.util.validate(size(foundIDs) < size(allInputIDs), apoc.convert.toJson({code: 404, ids: foundIDs}), null)\n\n// Check if given TypeInstances are not already locked by others\nWITH *\nCALL {\n WITH *\n UNWIND $in AS item\n MATCH (tic:TypeInstance {id: item.id})\n WHERE tic.lockedBy IS NOT NULL AND (item.ownerID IS NULL OR tic.lockedBy <> item.ownerID)\n WITH collect(tic.id) as lockedIDs\n RETURN lockedIDs\n}\nCALL apoc.util.validate(size(lockedIDs) > 0, apoc.convert.toJson({code: 409, ids: lockedIDs}), null)\n\nUNWIND $in as item\nMATCH (ti: TypeInstance {id: item.id})\nCALL {\n WITH ti\n MATCH (ti)-[:CONTAINS]->(latestRevision:TypeInstanceResourceVersion)\n RETURN latestRevision\n ORDER BY latestRevision.resourceVersion DESC LIMIT 1\n}\n\nCREATE (tir: TypeInstanceResourceVersion {resourceVersion: latestRevision.resourceVersion + 1, createdBy: item.createdBy})\nCREATE (ti)-[:CONTAINS]->(tir)\n\n// Handle the `spec.value` property\nCREATE (spec: TypeInstanceResourceVersionSpec)\nCREATE (tir)-[:SPECIFIED_BY]->(spec)\n\nWITH ti, tir, spec, latestRevision, item\nMATCH (ti)-[:STORED_IN]->(storageRef:TypeInstanceBackendReference)\n\nWITH ti, tir, spec, latestRevision, item, storageRef\nCALL apoc.do.case([\n storageRef.abstract AND item.typeInstance.value IS NOT NULL, // built-in: store new value\n '\n SET spec.value = apoc.convert.toJson(item.typeInstance.value) RETURN spec\n ',\n storageRef.abstract AND item.typeInstance.value IS NULL, // built-in: no value, so use old one\n '\n MATCH (latestRevision)-[:SPECIFIED_BY]->(latestSpec: TypeInstanceResourceVersionSpec)\n SET spec.value = latestSpec.value RETURN spec\n '\n ],\n '\n RETURN spec // external storage, do nothing\n ',\n{spec:spec, latestRevision: latestRevision, item: item}) YIELD value\n\n// Handle the `backend.context`\nWITH ti, tir, spec, latestRevision, item\nCALL apoc.do.when(\n item.typeInstance.backend IS NOT NULL,\n '\n CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: apoc.convert.toJson(item.typeInstance.backend.context)})\n RETURN specBackend\n ',\n '\n CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: apoc.convert.toJson(item.typeInstance.backend.context)})\n RETURN specBackend\n ',\n{spec:spec, latestRevision: latestRevision, item: item}) YIELD value as backendRef\nWITH ti, tir, spec, latestRevision, item, backendRef.specBackend as specBackend\nCREATE (spec)-[:WITH_BACKEND]->(specBackend)\n\n// Handle the `metadata.attributes` property\nCREATE (metadata: TypeInstanceResourceVersionMetadata)\nCREATE (tir)-[:DESCRIBED_BY]->(metadata)\n\nWITH ti, tir, latestRevision, metadata, item\nCALL apoc.do.when(\n item.typeInstance.attributes IS NOT NULL,\n '\n FOREACH (attr in item.typeInstance.attributes |\n MERGE (attrRef: AttributeReference {path: attr.path, revision: attr.revision})\n CREATE (metadata)-[:CHARACTERIZED_BY]->(attrRef)\n )\n\n RETURN metadata\n ',\n '\n OPTIONAL MATCH (latestRevision)-[:DESCRIBED_BY]->(TypeInstanceResourceVersionMetadata)-[:CHARACTERIZED_BY]->(latestAttrRef: AttributeReference)\n WHERE latestAttrRef IS NOT NULL\n WITH *, COLLECT(latestAttrRef) AS latestAttrRefs\n FOREACH (attr in latestAttrRefs |\n CREATE (metadata)-[:CHARACTERIZED_BY]->(attr)\n )\n\n RETURN metadata\n ',\n {metadata: metadata, latestRevision: latestRevision, item: item}\n) YIELD value\n\nRETURN ti") + statement, err := ec.unmarshalOString2áš–string(ctx, "CALL {\n UNWIND $in AS item\n RETURN collect(item.id) as allInputIDs\n}\n\n// Check if all TypeInstances were found\nWITH *\nCALL {\n WITH allInputIDs\n MATCH (ti:TypeInstance)\n WHERE ti.id IN allInputIDs\n WITH collect(ti.id) as foundIDs\n RETURN foundIDs\n}\nCALL apoc.util.validate(size(foundIDs) < size(allInputIDs), apoc.convert.toJson({code: 404, ids: foundIDs}), null)\n\n// Check if given TypeInstances are not already locked by others\nWITH *\nCALL {\n WITH *\n UNWIND $in AS item\n MATCH (tic:TypeInstance {id: item.id})\n WHERE tic.lockedBy IS NOT NULL AND (item.ownerID IS NULL OR tic.lockedBy <> item.ownerID)\n WITH collect(tic.id) as lockedIDs\n RETURN lockedIDs\n}\nCALL apoc.util.validate(size(lockedIDs) > 0, apoc.convert.toJson({code: 409, ids: lockedIDs}), null)\n\nUNWIND $in as item\nMATCH (ti: TypeInstance {id: item.id})\nCALL {\n WITH ti\n MATCH (ti)-[:CONTAINS]->(latestRevision:TypeInstanceResourceVersion)\n RETURN latestRevision\n ORDER BY latestRevision.resourceVersion DESC LIMIT 1\n}\n\nCREATE (tir: TypeInstanceResourceVersion {resourceVersion: latestRevision.resourceVersion + 1, createdBy: item.createdBy})\nCREATE (ti)-[:CONTAINS]->(tir)\n\n// Handle the `spec.value` property\nCREATE (spec: TypeInstanceResourceVersionSpec)\nCREATE (tir)-[:SPECIFIED_BY]->(spec)\n\nWITH ti, tir, spec, latestRevision, item\nMATCH (ti)-[:STORED_IN]->(storageRef:TypeInstanceBackendReference)\n\nWITH ti, tir, spec, latestRevision, item, storageRef\nCALL apoc.do.case([\n storageRef.abstract AND item.typeInstance.value IS NOT NULL, // built-in: store new value\n '\n SET spec.value = apoc.convert.toJson(item.typeInstance.value) RETURN spec\n ',\n storageRef.abstract AND item.typeInstance.value IS NULL, // built-in: no value, so use old one\n '\n MATCH (latestRevision)-[:SPECIFIED_BY]->(latestSpec: TypeInstanceResourceVersionSpec)\n SET spec.value = latestSpec.value RETURN spec\n '\n ],\n '\n RETURN spec // external storage, do nothing\n ',\n{spec:spec, latestRevision: latestRevision, item: item}) YIELD value\n\n// Handle the `backend.context`\nWITH ti, tir, spec, latestRevision, item\nCALL apoc.do.when(\n item.typeInstance.backend IS NOT NULL,\n '\n CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: apoc.convert.toJson(item.typeInstance.backend.context)})\n RETURN specBackend\n ',\n '\n MATCH (latestRevision)-[:SPECIFIED_BY]->(latestSpec: TypeInstanceResourceVersionSpec)\n MATCH (latestSpec)-[:WITH_BACKEND]->(oldSpecBackend:TypeInstanceResourceVersionSpecBackend)\n CREATE (specBackend: TypeInstanceResourceVersionSpecBackend {context: oldSpecBackend.context})\n RETURN specBackend\n ',\n{spec:spec, latestRevision: latestRevision, item: item}) YIELD value as backendRef\nWITH ti, tir, spec, latestRevision, item, backendRef.specBackend as specBackend\nCREATE (spec)-[:WITH_BACKEND]->(specBackend)\n\n// Handle the `metadata.attributes` property\nCREATE (metadata: TypeInstanceResourceVersionMetadata)\nCREATE (tir)-[:DESCRIBED_BY]->(metadata)\n\nWITH ti, tir, latestRevision, metadata, item\nCALL apoc.do.when(\n item.typeInstance.attributes IS NOT NULL,\n '\n FOREACH (attr in item.typeInstance.attributes |\n MERGE (attrRef: AttributeReference {path: attr.path, revision: attr.revision})\n CREATE (metadata)-[:CHARACTERIZED_BY]->(attrRef)\n )\n\n RETURN metadata\n ',\n '\n OPTIONAL MATCH (latestRevision)-[:DESCRIBED_BY]->(TypeInstanceResourceVersionMetadata)-[:CHARACTERIZED_BY]->(latestAttrRef: AttributeReference)\n WHERE latestAttrRef IS NOT NULL\n WITH *, COLLECT(latestAttrRef) AS latestAttrRefs\n FOREACH (attr in latestAttrRefs |\n CREATE (metadata)-[:CHARACTERIZED_BY]->(attr)\n )\n\n RETURN metadata\n ',\n {metadata: metadata, latestRevision: latestRevision, item: item}\n) YIELD value\n\nRETURN ti") if err != nil { return nil, err } diff --git a/pkg/hub/client/local/client_test.go b/pkg/hub/client/local/client_test.go new file mode 100644 index 000000000..4edaf5eea --- /dev/null +++ b/pkg/hub/client/local/client_test.go @@ -0,0 +1,325 @@ +package local + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + "capact.io/capact/internal/cli/heredoc" + cliprinter "capact.io/capact/internal/cli/printer" + "capact.io/capact/internal/ptr" + gqllocalapi "capact.io/capact/pkg/hub/api/graphql/local" + pb "capact.io/capact/pkg/hub/api/grpc/storage_backend" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// TODO(review): THIS FILE WILL BE REMOVED BEFORE MERGING. IT WAS ADDED ONLY FOR DEMO/PR TESTING PURPOSES. + +// #nosec G101 +const secretStorageBackendAddr = "GRPC_SECRET_STORAGE_BACKEND_ADDR" + +type StorageValue struct { + URL string `json:"url"` + AcceptValue bool `json:"acceptValue"` + ContextSchema string `json:"contextSchema"` +} + +// To run this test, execute: +// GRPC_SECRET_STORAGE_BACKEND_ADDR="0.0.0.0:50051" go test ./pkg/hub/client/local/ -v -count 1 +func TestThatShowcaseExternalStorage(t *testing.T) { + srvAddr := os.Getenv(secretStorageBackendAddr) + if srvAddr == "" { + t.Skipf("skipping running example test as the env %s is not provided", secretStorageBackendAddr) + } + + ctx := context.Background() + cli := NewDefaultClient("http://localhost:8080/graphql") + dotenvHubStorage, cleanup := registerExternalDotenvStorage(ctx, t, cli, srvAddr) + defer cleanup() + + // SCENARIO - CREATE + family, err := cli.CreateTypeInstances(ctx, &gqllocalapi.CreateTypeInstancesInput{ + TypeInstances: []*gqllocalapi.CreateTypeInstanceInput{ + { + // This TypeInstance: + // - is stored in built-in backend + // - doesn't have backend context + Alias: ptr.String("child"), + CreatedBy: ptr.String("nature"), + TypeRef: typeRef("cap.type.child:0.1.0"), + Value: map[string]interface{}{ + "name": "Luke Skywalker", + }, + }, + { + // This TypeInstance: + // - is stored in external backend + // - context won't be updated in next scenario, so the old one should be used + Alias: ptr.String("original"), + CreatedBy: ptr.String("nature"), + TypeRef: typeRef("cap.type.original:0.2.0"), + Value: map[string]interface{}{ + "name": "Anakin Skywalker", + }, + Backend: &gqllocalapi.TypeInstanceBackendInput{ + ID: dotenvHubStorage.ID, + Context: map[string]interface{}{ + "provider": "dotenv", + }, + }, + }, + { + // This TypeInstance: + // - is stored in external backend + // - context will be updated, new one should be used + Alias: ptr.String("parent"), + CreatedBy: ptr.String("nature"), + TypeRef: typeRef("cap.type.parent:0.1.0"), + Value: map[string]interface{}{ + "name": "Darth Vader", + }, + Backend: &gqllocalapi.TypeInstanceBackendInput{ + ID: dotenvHubStorage.ID, + Context: map[string]interface{}{ + "provider": "dotenv", + }, + }, + }, + }, + UsesRelations: []*gqllocalapi.TypeInstanceUsesRelationInput{ + {From: "parent", To: "child"}, + {From: "parent", To: "original"}, + }, + }) + require.NoError(t, err) + + familyDetails, err := cli.ListTypeInstances(ctx, &gqllocalapi.TypeInstanceFilter{ + CreatedBy: ptr.String("nature"), + }, WithFields(TypeInstanceAllFields)) + require.NoError(t, err) + + defer removeAllMembers(t, cli, familyDetails) + + fmt.Print("\n\n======== After create result ============\n\n") + resourcePrinter := cliprinter.NewForResource(os.Stdout, cliprinter.WithTable(typeInstanceDetailsMapper(family, getDataDirectlyFromStorage(t, srvAddr, familyDetails)))) + require.NoError(t, resourcePrinter.Print(familyDetails)) + + // SCENARIO - UPDATE + // - for cap.type.parent don't update `value` and `context` - use old ones + // - for cap.type.original:0.2.0 don't update `value`, and zero the `context` + // - for all others, update both `value` and `context` + toUpdate := make([]gqllocalapi.UpdateTypeInstancesInput, 0, len(familyDetails)) + for idx, member := range familyDetails { + val := map[string]interface{}{ + "updated-value": fmt.Sprintf("context %d", idx), + } + backend := &gqllocalapi.UpdateTypeInstanceBackendInput{ + Context: map[string]interface{}{ + "updated": fmt.Sprintf("context %d", idx), + }, + } + // For cap.type.original:0.2.0 don't update value, and zero the `context`. + if member.TypeRef.Path == "cap.type.original" { + val = nil + backend.Context = nil + } + + // For cap.type.parent don't update value and `context` - use old ones + if member.TypeRef.Path == "cap.type.parent" { + val = nil + backend = nil + } + toUpdate = append(toUpdate, gqllocalapi.UpdateTypeInstancesInput{ + ID: member.ID, + CreatedBy: ptr.String("update"), + TypeInstance: &gqllocalapi.UpdateTypeInstanceInput{ + Value: val, + Backend: backend, + }, + }) + } + + updatedFamily, err := cli.UpdateTypeInstances(ctx, toUpdate) + require.NoError(t, err) + + fmt.Print("\n\n======== After update result ============\n\n") + resourcePrinter = cliprinter.NewForResource(os.Stdout, cliprinter.WithTable(typeInstanceDetailsMapper(family, getDataDirectlyFromStorage(t, srvAddr, updatedFamily)))) + require.NoError(t, resourcePrinter.Print(updatedFamily)) +} + +// ======= HELPERS ======= + +func registerExternalDotenvStorage(ctx context.Context, t *testing.T, cli *Client, srvAddr string) (gqllocalapi.CreateTypeInstanceOutput, func()) { + t.Helper() + + ti, err := cli.CreateTypeInstances(ctx, fixExternalDotenvStorage(srvAddr)) + require.NoError(t, err) + require.Len(t, ti, 1) + dotenvHubStorage := ti[0] + + return dotenvHubStorage, func() { + _ = cli.DeleteTypeInstance(ctx, dotenvHubStorage.ID) + } +} + +func fixExternalDotenvStorage(addr string) *gqllocalapi.CreateTypeInstancesInput { + return &gqllocalapi.CreateTypeInstancesInput{ + TypeInstances: []*gqllocalapi.CreateTypeInstanceInput{ + { + CreatedBy: ptr.String("manually"), + TypeRef: &gqllocalapi.TypeInstanceTypeReferenceInput{ + Path: "cap.type.example.filesystem.storage", + Revision: "0.1.0", + }, + Value: StorageValue{ + URL: addr, + AcceptValue: true, + ContextSchema: heredoc.Doc(` + { + "$id": "#/properties/contextSchema", + "type": "object", + "properties": { + "provider": { + "$id": "#/properties/contextSchema/properties/name", + "type": "string", + "const": "dotenv" + } + }, + "additionalProperties": false + }`), + }, + }, + }, + UsesRelations: []*gqllocalapi.TypeInstanceUsesRelationInput{}, + } +} + +type externalData struct { + Value string + LockedBy *string +} + +func getDataDirectlyFromStorage(t *testing.T, addr string, details []gqllocalapi.TypeInstance) map[string]externalData { + t.Helper() + + conn, err := grpc.Dial(addr, grpc.WithInsecure()) + require.NoError(t, err) + + ctx := context.Background() + client := pb.NewStorageBackendClient(conn) + + var out = map[string]externalData{} + for _, ti := range details { + val, err := client.GetValue(ctx, &pb.GetValueRequest{ + TypeInstanceId: ti.ID, + ResourceVersion: uint32(ti.LatestResourceVersion.ResourceVersion), + }) + if err != nil { + continue + } + + locked, err := client.GetLockedBy(ctx, &pb.GetLockedByRequest{ + TypeInstanceId: ti.ID, + }) + if err != nil { + continue + } + + out[ti.ID] = externalData{ + Value: string(val.Value), + LockedBy: locked.LockedBy, + } + } + return out +} + +func typeInstanceDetailsMapper(family []gqllocalapi.CreateTypeInstanceOutput, storage map[string]externalData) func(inRaw interface{}) (cliprinter.TableData, error) { + mapping := map[string]string{} + for _, member := range family { + mapping[member.ID] = member.Alias + } + labelIfAbstract := func(in bool) string { + if in { + return " (abstract)" + } + return "" + } + return func(inRaw interface{}) (cliprinter.TableData, error) { + out := cliprinter.TableData{} + + switch in := inRaw.(type) { + case []gqllocalapi.TypeInstance: + out.Headers = []string{"TYPE INSTANCE ID", "ALIAS", "TYPE", "BACKEND", "BACKEND CONTEXT", "DATA IN GQL", "DATA IN exBACKEND", "LOCKED", "LOCKED IN exBACKEND"} + for _, ti := range in { + out.MultipleRows = append(out.MultipleRows, []string{ + ti.ID, + mapping[ti.ID], + fmt.Sprintf("%s:%s", ti.TypeRef.Path, ti.TypeRef.Revision), + fmt.Sprintf("%s%s", ti.Backend.ID, labelIfAbstract(ti.Backend.Abstract)), + mustMarshal(ti.LatestResourceVersion.Spec.Backend.Context), + mustMarshal(ti.LatestResourceVersion.Spec.Value), + storage[ti.ID].Value, + stringDefault(ti.LockedBy, "-"), + stringDefault(storage[ti.ID].LockedBy, "-"), + }) + } + default: + return cliprinter.TableData{}, fmt.Errorf("got unexpected input type, expected []gqllocalapi.TypeInstance, got %T", inRaw) + } + + return out, nil + } +} + +func mustMarshal(v interface{}) string { + out, err := json.Marshal(v) + if err != nil { + panic(err) + } + return string(out) +} + +func stringDefault(in *string, def string) string { + if in == nil { + return def + } + return *in +} + +func typeRef(in string) *gqllocalapi.TypeInstanceTypeReferenceInput { + out := strings.Split(in, ":") + return &gqllocalapi.TypeInstanceTypeReferenceInput{Path: out[0], Revision: out[1]} +} + +func removeAllMembers(t *testing.T, cli *Client, familyDetails []gqllocalapi.TypeInstance) { + t.Helper() + + ctx := context.Background() + + for _, member := range familyDetails { + if member.TypeRef.Path != "cap.type.parent" { + defer func(id string) { // delay the child deletions + fmt.Println("Delete child", id) + err := cli.DeleteTypeInstance(ctx, id) + if err != nil { + t.Logf("err for %v: %v", id, err) + } + }(member.ID) + + continue + } + + fmt.Println("Delete parent", member.ID) + + // Delete parent first, to unblock deletion of children + err := cli.DeleteTypeInstance(ctx, member.ID) + if err != nil { + t.Logf("err for %v: %v", member.ID, err) + } + } +} From 20a0de72df46cdb1c00f41d8b2f92fa38803e958 Mon Sep 17 00:00:00 2001 From: Mateusz Szostok Date: Thu, 17 Mar 2022 08:44:54 +0100 Subject: [PATCH 2/2] Remove tests --- pkg/hub/client/local/client_test.go | 325 ---------------------------- 1 file changed, 325 deletions(-) delete mode 100644 pkg/hub/client/local/client_test.go diff --git a/pkg/hub/client/local/client_test.go b/pkg/hub/client/local/client_test.go deleted file mode 100644 index 4edaf5eea..000000000 --- a/pkg/hub/client/local/client_test.go +++ /dev/null @@ -1,325 +0,0 @@ -package local - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strings" - "testing" - - "capact.io/capact/internal/cli/heredoc" - cliprinter "capact.io/capact/internal/cli/printer" - "capact.io/capact/internal/ptr" - gqllocalapi "capact.io/capact/pkg/hub/api/graphql/local" - pb "capact.io/capact/pkg/hub/api/grpc/storage_backend" - - "github.com/stretchr/testify/require" - "google.golang.org/grpc" -) - -// TODO(review): THIS FILE WILL BE REMOVED BEFORE MERGING. IT WAS ADDED ONLY FOR DEMO/PR TESTING PURPOSES. - -// #nosec G101 -const secretStorageBackendAddr = "GRPC_SECRET_STORAGE_BACKEND_ADDR" - -type StorageValue struct { - URL string `json:"url"` - AcceptValue bool `json:"acceptValue"` - ContextSchema string `json:"contextSchema"` -} - -// To run this test, execute: -// GRPC_SECRET_STORAGE_BACKEND_ADDR="0.0.0.0:50051" go test ./pkg/hub/client/local/ -v -count 1 -func TestThatShowcaseExternalStorage(t *testing.T) { - srvAddr := os.Getenv(secretStorageBackendAddr) - if srvAddr == "" { - t.Skipf("skipping running example test as the env %s is not provided", secretStorageBackendAddr) - } - - ctx := context.Background() - cli := NewDefaultClient("http://localhost:8080/graphql") - dotenvHubStorage, cleanup := registerExternalDotenvStorage(ctx, t, cli, srvAddr) - defer cleanup() - - // SCENARIO - CREATE - family, err := cli.CreateTypeInstances(ctx, &gqllocalapi.CreateTypeInstancesInput{ - TypeInstances: []*gqllocalapi.CreateTypeInstanceInput{ - { - // This TypeInstance: - // - is stored in built-in backend - // - doesn't have backend context - Alias: ptr.String("child"), - CreatedBy: ptr.String("nature"), - TypeRef: typeRef("cap.type.child:0.1.0"), - Value: map[string]interface{}{ - "name": "Luke Skywalker", - }, - }, - { - // This TypeInstance: - // - is stored in external backend - // - context won't be updated in next scenario, so the old one should be used - Alias: ptr.String("original"), - CreatedBy: ptr.String("nature"), - TypeRef: typeRef("cap.type.original:0.2.0"), - Value: map[string]interface{}{ - "name": "Anakin Skywalker", - }, - Backend: &gqllocalapi.TypeInstanceBackendInput{ - ID: dotenvHubStorage.ID, - Context: map[string]interface{}{ - "provider": "dotenv", - }, - }, - }, - { - // This TypeInstance: - // - is stored in external backend - // - context will be updated, new one should be used - Alias: ptr.String("parent"), - CreatedBy: ptr.String("nature"), - TypeRef: typeRef("cap.type.parent:0.1.0"), - Value: map[string]interface{}{ - "name": "Darth Vader", - }, - Backend: &gqllocalapi.TypeInstanceBackendInput{ - ID: dotenvHubStorage.ID, - Context: map[string]interface{}{ - "provider": "dotenv", - }, - }, - }, - }, - UsesRelations: []*gqllocalapi.TypeInstanceUsesRelationInput{ - {From: "parent", To: "child"}, - {From: "parent", To: "original"}, - }, - }) - require.NoError(t, err) - - familyDetails, err := cli.ListTypeInstances(ctx, &gqllocalapi.TypeInstanceFilter{ - CreatedBy: ptr.String("nature"), - }, WithFields(TypeInstanceAllFields)) - require.NoError(t, err) - - defer removeAllMembers(t, cli, familyDetails) - - fmt.Print("\n\n======== After create result ============\n\n") - resourcePrinter := cliprinter.NewForResource(os.Stdout, cliprinter.WithTable(typeInstanceDetailsMapper(family, getDataDirectlyFromStorage(t, srvAddr, familyDetails)))) - require.NoError(t, resourcePrinter.Print(familyDetails)) - - // SCENARIO - UPDATE - // - for cap.type.parent don't update `value` and `context` - use old ones - // - for cap.type.original:0.2.0 don't update `value`, and zero the `context` - // - for all others, update both `value` and `context` - toUpdate := make([]gqllocalapi.UpdateTypeInstancesInput, 0, len(familyDetails)) - for idx, member := range familyDetails { - val := map[string]interface{}{ - "updated-value": fmt.Sprintf("context %d", idx), - } - backend := &gqllocalapi.UpdateTypeInstanceBackendInput{ - Context: map[string]interface{}{ - "updated": fmt.Sprintf("context %d", idx), - }, - } - // For cap.type.original:0.2.0 don't update value, and zero the `context`. - if member.TypeRef.Path == "cap.type.original" { - val = nil - backend.Context = nil - } - - // For cap.type.parent don't update value and `context` - use old ones - if member.TypeRef.Path == "cap.type.parent" { - val = nil - backend = nil - } - toUpdate = append(toUpdate, gqllocalapi.UpdateTypeInstancesInput{ - ID: member.ID, - CreatedBy: ptr.String("update"), - TypeInstance: &gqllocalapi.UpdateTypeInstanceInput{ - Value: val, - Backend: backend, - }, - }) - } - - updatedFamily, err := cli.UpdateTypeInstances(ctx, toUpdate) - require.NoError(t, err) - - fmt.Print("\n\n======== After update result ============\n\n") - resourcePrinter = cliprinter.NewForResource(os.Stdout, cliprinter.WithTable(typeInstanceDetailsMapper(family, getDataDirectlyFromStorage(t, srvAddr, updatedFamily)))) - require.NoError(t, resourcePrinter.Print(updatedFamily)) -} - -// ======= HELPERS ======= - -func registerExternalDotenvStorage(ctx context.Context, t *testing.T, cli *Client, srvAddr string) (gqllocalapi.CreateTypeInstanceOutput, func()) { - t.Helper() - - ti, err := cli.CreateTypeInstances(ctx, fixExternalDotenvStorage(srvAddr)) - require.NoError(t, err) - require.Len(t, ti, 1) - dotenvHubStorage := ti[0] - - return dotenvHubStorage, func() { - _ = cli.DeleteTypeInstance(ctx, dotenvHubStorage.ID) - } -} - -func fixExternalDotenvStorage(addr string) *gqllocalapi.CreateTypeInstancesInput { - return &gqllocalapi.CreateTypeInstancesInput{ - TypeInstances: []*gqllocalapi.CreateTypeInstanceInput{ - { - CreatedBy: ptr.String("manually"), - TypeRef: &gqllocalapi.TypeInstanceTypeReferenceInput{ - Path: "cap.type.example.filesystem.storage", - Revision: "0.1.0", - }, - Value: StorageValue{ - URL: addr, - AcceptValue: true, - ContextSchema: heredoc.Doc(` - { - "$id": "#/properties/contextSchema", - "type": "object", - "properties": { - "provider": { - "$id": "#/properties/contextSchema/properties/name", - "type": "string", - "const": "dotenv" - } - }, - "additionalProperties": false - }`), - }, - }, - }, - UsesRelations: []*gqllocalapi.TypeInstanceUsesRelationInput{}, - } -} - -type externalData struct { - Value string - LockedBy *string -} - -func getDataDirectlyFromStorage(t *testing.T, addr string, details []gqllocalapi.TypeInstance) map[string]externalData { - t.Helper() - - conn, err := grpc.Dial(addr, grpc.WithInsecure()) - require.NoError(t, err) - - ctx := context.Background() - client := pb.NewStorageBackendClient(conn) - - var out = map[string]externalData{} - for _, ti := range details { - val, err := client.GetValue(ctx, &pb.GetValueRequest{ - TypeInstanceId: ti.ID, - ResourceVersion: uint32(ti.LatestResourceVersion.ResourceVersion), - }) - if err != nil { - continue - } - - locked, err := client.GetLockedBy(ctx, &pb.GetLockedByRequest{ - TypeInstanceId: ti.ID, - }) - if err != nil { - continue - } - - out[ti.ID] = externalData{ - Value: string(val.Value), - LockedBy: locked.LockedBy, - } - } - return out -} - -func typeInstanceDetailsMapper(family []gqllocalapi.CreateTypeInstanceOutput, storage map[string]externalData) func(inRaw interface{}) (cliprinter.TableData, error) { - mapping := map[string]string{} - for _, member := range family { - mapping[member.ID] = member.Alias - } - labelIfAbstract := func(in bool) string { - if in { - return " (abstract)" - } - return "" - } - return func(inRaw interface{}) (cliprinter.TableData, error) { - out := cliprinter.TableData{} - - switch in := inRaw.(type) { - case []gqllocalapi.TypeInstance: - out.Headers = []string{"TYPE INSTANCE ID", "ALIAS", "TYPE", "BACKEND", "BACKEND CONTEXT", "DATA IN GQL", "DATA IN exBACKEND", "LOCKED", "LOCKED IN exBACKEND"} - for _, ti := range in { - out.MultipleRows = append(out.MultipleRows, []string{ - ti.ID, - mapping[ti.ID], - fmt.Sprintf("%s:%s", ti.TypeRef.Path, ti.TypeRef.Revision), - fmt.Sprintf("%s%s", ti.Backend.ID, labelIfAbstract(ti.Backend.Abstract)), - mustMarshal(ti.LatestResourceVersion.Spec.Backend.Context), - mustMarshal(ti.LatestResourceVersion.Spec.Value), - storage[ti.ID].Value, - stringDefault(ti.LockedBy, "-"), - stringDefault(storage[ti.ID].LockedBy, "-"), - }) - } - default: - return cliprinter.TableData{}, fmt.Errorf("got unexpected input type, expected []gqllocalapi.TypeInstance, got %T", inRaw) - } - - return out, nil - } -} - -func mustMarshal(v interface{}) string { - out, err := json.Marshal(v) - if err != nil { - panic(err) - } - return string(out) -} - -func stringDefault(in *string, def string) string { - if in == nil { - return def - } - return *in -} - -func typeRef(in string) *gqllocalapi.TypeInstanceTypeReferenceInput { - out := strings.Split(in, ":") - return &gqllocalapi.TypeInstanceTypeReferenceInput{Path: out[0], Revision: out[1]} -} - -func removeAllMembers(t *testing.T, cli *Client, familyDetails []gqllocalapi.TypeInstance) { - t.Helper() - - ctx := context.Background() - - for _, member := range familyDetails { - if member.TypeRef.Path != "cap.type.parent" { - defer func(id string) { // delay the child deletions - fmt.Println("Delete child", id) - err := cli.DeleteTypeInstance(ctx, id) - if err != nil { - t.Logf("err for %v: %v", id, err) - } - }(member.ID) - - continue - } - - fmt.Println("Delete parent", member.ID) - - // Delete parent first, to unblock deletion of children - err := cli.DeleteTypeInstance(ctx, member.ID) - if err != nil { - t.Logf("err for %v: %v", member.ID, err) - } - } -}