Skip to content

Describe approach for dynamic TypeInstances in nested workflows and update gRPC API #696

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pkosiec merged 6 commits into from
Apr 6, 2022
Merged

Describe approach for dynamic TypeInstances in nested workflows and update gRPC API #696

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
049bc09
Add document
pkosiec Mar 30, 2022
d5aa829
WIP
pkosiec Mar 31, 2022
c0c3a2e
Add agreement
pkosiec Mar 31, 2022
082d93a
Update delegated storage proposal
pkosiec Apr 4, 2022
e5239b7
Update API
pkosiec Apr 4, 2022
123b218
Rename ContextOnlyStorageBackend to ContextStorageBackend
pkosiec Apr 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/helm-storage-backend/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func main() {
parallelServers := new(errgroup.Group)

// create handler
var handler storage_backend.StorageBackendServer
var handler storage_backend.ContextStorageBackendServer
switch cfg.Mode {
case HelmReleaseMode:
handler, err = helm_storage_backend.NewReleaseHandler(logger, relFetcher)
Expand All @@ -94,7 +94,7 @@ func main() {
exitOnError(err, "while listening")

srv := grpc.NewServer()
storage_backend.RegisterStorageBackendServer(srv, handler)
storage_backend.RegisterContextStorageBackendServer(srv, handler)

go func() {
<-ctx.Done()
Expand Down
2 changes: 1 addition & 1 deletion cmd/secret-storage-backend/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func main() {
exitOnError(err, "while listening")

srv := grpc.NewServer()
storage_backend.RegisterStorageBackendServer(srv, handler)
storage_backend.RegisterValueAndContextStorageBackendServer(srv, handler)

go func() {
<-ctx.Done()
Expand Down
109 changes: 84 additions & 25 deletions docs/proposal/20211207-delegated-storage.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ Also, the additional, nice-to-have goals are:
"$id": "#/properties/acceptValue",
"type": "boolean",
"const": false # in this case - no
},
}
},
"additionalProperties": false
}
Expand Down Expand Up @@ -296,6 +296,21 @@ Also, the additional, nice-to-have goals are:

However, the `context` are backend-specific properties, which means Content Developer need to explicitly specify the backend as described later.

Some further workflow steps might need actual values. To allow that, a dedicated workflow step can be used:

```yaml
- - name: resolve-dynamic-ti
template: resolve-dynamic-ti
arguments:
- name: input-artifact
from: "{{steps.resolve-dynamic-ti.outputs.artifacts.postgresql}}"
- name: backend
# Backend is already injected into workflow based on the `alias` from `spec.requires` property:
from: "{{workflow.outputs.artifacts.helm-template-storage}}"
```

That container fills the `value` property based on a given storage backend. However, the storage backend would need to support fetching value based purely on context, without TypeInstance ID, that is the `PreCreateValue` method. For more information, see the [Storage backend service implementation](#storage-backend-service-implementation).

1. To save a specific value with additional parameters:

For example, for an implementation of Kubernetes secrets storage backend, which actually creates and updates these secrets during TypeInstance creation:
Expand Down Expand Up @@ -415,21 +430,16 @@ Also, the additional, nice-to-have goals are:
1. Hub calls the registered storage backend service `onCreate` hook:

```proto
message TypeInstanceData {
string id = 1;
bytes value = 2;
}

message OnCreateRequest {
TypeInstanceData typeinstance = 1;
string type_instance_id = 1;
bytes context = 2;
}

message OnCreateResponse {
optional bytes context = 1;
}

service SearchService {
service ContextStorageBackend {
rpc OnCreate(OnCreateRequest) returns (OnCreateResponse);
}
```
Expand Down Expand Up @@ -480,10 +490,23 @@ Capact Local Hub calls proper storage backend service while accessing the TypeIn
option go_package = "./storage_backend";
package storage_backend;

message GetPreCreateValueRequest {
bytes context = 1;
}

message GetPreCreateValueResponse {
optional bytes value = 1;
}

message OnCreateRequest {
string typeinstance_id = 1;
string type_instance_id = 1;
bytes context = 2;
}

message OnCreateValueAndContextRequest {
string type_instance_id = 1;
bytes value = 2;
bytes context = 3;
optional bytes context = 3;
}

message OnCreateResponse {
Expand All @@ -495,26 +518,41 @@ Capact Local Hub calls proper storage backend service while accessing the TypeIn
bytes value = 2;
}

message OnUpdateRequest {
string typeinstance_id = 1;
message OnUpdateValueAndContextRequest {
string type_instance_id = 1;
uint32 new_resource_version = 2;
bytes new_value = 3;
optional bytes context = 4;
optional string owner_id = 5;
}

message OnUpdateRequest {
string type_instance_id = 1;
uint32 new_resource_version = 2;
bytes context = 3;
optional string owner_id = 4;
}

message OnUpdateResponse {
optional bytes context = 1;
}

message OnDeleteValueAndContextRequest {
string type_instance_id = 1;
optional bytes context = 2;
optional string owner_id = 3;
}

message OnDeleteRequest {
string typeinstance_id = 1;
string type_instance_id = 1;
bytes context = 2;
optional string owner_id = 3;
}

message OnDeleteResponse {}

message GetValueRequest {
string typeinstance_id = 1;
string type_instance_id = 1;
uint32 resource_version = 2;
bytes context = 3;
}
Expand All @@ -527,7 +565,7 @@ Capact Local Hub calls proper storage backend service while accessing the TypeIn
// lock messages

message GetLockedByRequest {
string typeinstance_id = 1;
string type_instance_id = 1;
bytes context = 2;
}

Expand All @@ -536,42 +574,59 @@ Capact Local Hub calls proper storage backend service while accessing the TypeIn
}

message OnLockRequest {
string typeinstance_id = 1;
string type_instance_id = 1;
bytes context = 2;
string locked_by = 3;
}

message OnLockResponse {}

message OnUnlockRequest {
string typeinstance_id = 1;
string type_instance_id = 1;
bytes context = 2;
}

message OnUnlockResponse {}

// services

service StorageBackend {
// ValueAndContextStorageBackend handles the full lifecycle of the TypeInstance.
// TypeInstance value is always provided as a part of request. Context may be provided but it is not required.
service ValueAndContextStorageBackend {
// value
rpc GetValue(GetValueRequest) returns (GetValueResponse);
rpc OnCreate(OnCreateValueAndContextRequest) returns (OnCreateResponse);
rpc OnUpdate(OnUpdateValueAndContextRequest) returns (OnUpdateResponse);
rpc OnDelete(OnDeleteValueAndContextRequest) returns (OnDeleteResponse);

// lock
rpc GetLockedBy(GetLockedByRequest) returns (GetLockedByResponse);
rpc OnLock(OnLockRequest) returns (OnLockResponse);
rpc OnUnlock(OnUnlockRequest) returns (OnUnlockResponse);
}

// ContextStorageBackend handles TypeInstance lifecycle based on the context, which is required. TypeInstance value is never passed in input arguments.
service ContextStorageBackend {
//value
rpc GetPreCreateValue(GetPreCreateValueRequest) returns (GetPreCreateValueResponse);
rpc GetValue(GetValueRequest) returns (GetValueResponse);
rpc OnCreate(OnCreateRequest) returns (OnCreateResponse);
rpc OnUpdate(OnUpdateRequest) returns (OnUpdateResponse);
rpc OnDelete(OnDeleteRequest) returns (OnDeleteResponse);

// lock
rpc GetLockedBy(GetLockedByRequest) returns (GetLockedByResponse);
rpc OnLock(OnLockRequest) returns (OnLockRequest);
rpc OnUnlock(OnUnlockRequest) returns (OnUnlockRequest);
rpc OnLock(OnLockRequest) returns (OnLockResponse);
rpc OnUnlock(OnUnlockRequest) returns (OnUnlockResponse);
}
```

</details>

An implementation of such service may vary between two use cases:
The service may implement a different API, depending on a given use case:

1. CRUD operations on output TypeInstance actually manages external resource (e.g. Vault) -> onCreate, onUpdate, and onDelete actually creates, updates and deletes a given resource.
1. output TypeInstance represents external resources managed in different way (e.g. via Capact actions - like Helm Runner). IMO we shouldn't move actual Helm release installation to TypeInstance "constructor").
1. `ValueAndContextStorageBackend`: CRUD operations on output TypeInstance actually manages external resource (e.g. Vault) -> onCreate, onUpdate, and onDelete actually creates, updates and deletes a given resource.
3. `ContextStorageBackend`: Output TypeInstance represents external resources managed in different way (e.g. via Capact actions - like Helm Runner). IMO we shouldn't move actual Helm release installation to TypeInstance "constructor").

- The service can also implement watch for external resources (e.g. Kubernetes secrets) and call `createTypeInstances` and `deleteTypeInstances` Hub mutations. We may provide Go framework to speed up such development, similarly as we have with Runner concept.

Expand Down Expand Up @@ -933,11 +988,11 @@ To avoid implementing a special storage backend service every time we have such
}
}
},
"acceptValue": { # specifies if a given storage backend (app) accepts TypeInstance value while creating/updating TypeInstance, or just context.
"acceptValue": { # specifies if a given storage backend (app) accepts TypeInstance value while creating/updating TypeInstance, or just context
"$id": "#/properties/acceptValue",
"type": "boolean",
"const": false # in this case - no
},
}
},
"additionalProperties": false
}
Expand Down Expand Up @@ -1228,6 +1283,7 @@ Once approved, we need to address the following list of items:
- Extend `capact-outputTypeInstances` syntax
- Set proper `uses` relations between storage backend TypeInstance and other TypeInstances
- Modify TypeInstance create/update/delete images (named as "Argo actions") to take new input
- ignore TypeInstance value if a given Storage Backend is dynamic
1. Update Policy
- Add new properties
- Handle common TypeInstance injections
Expand All @@ -1238,3 +1294,6 @@ Once approved, we need to address the following list of items:
1. Runners
- Remove `output.goTemplate`
- Stop supporting usage of funcs from `_helpers.tpl` in case of Helm runner
1. Support unpacking value by Jinja2 and Artifact Merger
1. Add generic container to enrich TypeInstance (run `GetPreCreateValue`) at the end of a given workflow
- Manually added by Content Developer
49 changes: 46 additions & 3 deletions hub-js/proto/storage_backend.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,20 @@ syntax = "proto3";
option go_package = "./storage_backend";
package storage_backend;

message GetPreCreateValueRequest {
bytes context = 1;
}

message GetPreCreateValueResponse {
optional bytes value = 1;
}

message OnCreateRequest {
string type_instance_id = 1;
bytes context = 2;
}

message OnCreateValueAndContextRequest {
string type_instance_id = 1;
bytes value = 2;
optional bytes context = 3;
Expand All @@ -17,24 +30,37 @@ message TypeInstanceResourceVersion {
bytes value = 2;
}

message OnUpdateRequest {
message OnUpdateValueAndContextRequest {
string type_instance_id = 1;
uint32 new_resource_version = 2;
bytes new_value = 3;
optional bytes context = 4;
optional string owner_id = 5;
}

message OnUpdateRequest {
string type_instance_id = 1;
uint32 new_resource_version = 2;
bytes context = 3;
optional string owner_id = 4;
}

message OnUpdateResponse {
optional bytes context = 1;
}

message OnDeleteRequest {
message OnDeleteValueAndContextRequest {
string type_instance_id = 1;
optional bytes context = 2;
optional string owner_id = 3;
}

message OnDeleteRequest {
string type_instance_id = 1;
bytes context = 2;
optional string owner_id = 3;
}

message OnDeleteResponse {}

message GetValueRequest {
Expand Down Expand Up @@ -76,9 +102,26 @@ message OnUnlockResponse {}

// services

service StorageBackend {
// ValueAndContextStorageBackend handles the full lifecycle of the TypeInstance.
// TypeInstance value is always provided as a part of request. Context may be provided but it is not required.
service ValueAndContextStorageBackend {
// value
rpc GetValue(GetValueRequest) returns (GetValueResponse);
rpc OnCreate(OnCreateValueAndContextRequest) returns (OnCreateResponse);
rpc OnUpdate(OnUpdateValueAndContextRequest) returns (OnUpdateResponse);
rpc OnDelete(OnDeleteValueAndContextRequest) returns (OnDeleteResponse);

// lock
rpc GetLockedBy(GetLockedByRequest) returns (GetLockedByResponse);
rpc OnLock(OnLockRequest) returns (OnLockResponse);
rpc OnUnlock(OnUnlockRequest) returns (OnUnlockResponse);
}

// ContextStorageBackend handles TypeInstance lifecycle based on the context, which is required. TypeInstance value is never passed in input arguments.
service ContextStorageBackend {
//value
rpc GetPreCreateValue(GetPreCreateValueRequest) returns (GetPreCreateValueResponse);
rpc GetValue(GetValueRequest) returns (GetValueResponse);
rpc OnCreate(OnCreateRequest) returns (OnCreateResponse);
rpc OnUpdate(OnUpdateRequest) returns (OnUpdateResponse);
rpc OnDelete(OnDeleteRequest) returns (OnDeleteResponse);
Expand Down
Loading