From db5ca0f67f2c6ba880c4f348372a12c71cf34d7f Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 16:08:54 +0200 Subject: [PATCH 01/15] fix: Fixed description for GetProjectCollectionIdsByPath --- aruna/api/storage/services/v1/object_service.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aruna/api/storage/services/v1/object_service.proto b/aruna/api/storage/services/v1/object_service.proto index b775b7eb..51abdbc9 100644 --- a/aruna/api/storage/services/v1/object_service.proto +++ b/aruna/api/storage/services/v1/object_service.proto @@ -342,11 +342,11 @@ service ObjectService { }; } - // GetObjectsByPath + // GetProjectCollectionIdsByPath // // Status: BETA // - // Gets a specific object by object_path + // Gets a specific project and collection_id by object_path // !! Paths are collection specific !! rpc GetProjectCollectionIdsByPath(GetProjectCollectionIdsByPathRequest) returns (GetProjectCollectionIdsByPathResponse) { option (google.api.http) = { From dc0a9446b65ff11d77f672dd609984afd8924d43 Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 16:24:52 +0200 Subject: [PATCH 02/15] feat: Added GetObjectsAsListV2Request --- .../storage/services/v1/object_service.proto | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/aruna/api/storage/services/v1/object_service.proto b/aruna/api/storage/services/v1/object_service.proto index 51abdbc9..8764b8c5 100644 --- a/aruna/api/storage/services/v1/object_service.proto +++ b/aruna/api/storage/services/v1/object_service.proto @@ -354,6 +354,19 @@ service ObjectService { }; } + // GetObjectsAsListV2 + // + // Status: ALPHA + // + // Gets a list of objects represented similar to a S3 ListObjectsV2 request + // !! Paths are collection specific !! + rpc GetObjectsAsListV2(GetObjectsAsListV2Request) returns (GetObjectsAsListV2Response) { + option (google.api.http) = { + get : "/v1/path/objects/listv2" + }; + } + + } // Models @@ -795,3 +808,30 @@ message GetObjectsByPathResponse { repeated storage.models.v1.Object object = 3; } + +message GetObjectsAsListV2Request { + // Required + string bucket = 1; + optional string continuation_token = 2; + optional string delimiter = 3; + optional uint32 max_keys = 4; + optional string prefix = 5; + optional string start_after = 6; +} + +message CommonPrefix { + string prefix = 1; +} + +message GetObjectsAsListV2Response { + string name = 1; + bool is_truncated = 2; + uint32 max_keys = 4; + uint32 key_count = 5; + repeated storage.models.v1.Object contents = 6; + repeated CommonPrefix prefixes = 7; + optional string next_continuation_token = 8; +} + + + From 2edfd126bb91492ee85e2eb1bb5269da127002f3 Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 16:44:37 +0200 Subject: [PATCH 03/15] feat: Added workspace API --- .../services/v1/collection_service.proto | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/aruna/api/storage/services/v1/collection_service.proto b/aruna/api/storage/services/v1/collection_service.proto index 296d8eee..7a3cb1b2 100644 --- a/aruna/api/storage/services/v1/collection_service.proto +++ b/aruna/api/storage/services/v1/collection_service.proto @@ -87,7 +87,7 @@ service CollectionService { rpc PinCollectionVersion(PinCollectionVersionRequest) returns (PinCollectionVersionResponse) { option (google.api.http) = { - post : "/v1/collection/{collection_id}/pin" + post : "/v1/workspace/" body : "*" }; } @@ -109,8 +109,99 @@ service CollectionService { } } + +// WorkspaceService +// +// Service to manage "special" anonymous collections / workspaces +service WorkspaceService { + // CreateWorkspace + // + // Status: ALPHA + // + // A new request to create a personal anonymous workspace + rpc CreateWorkspace(CreateWorkspaceRequest) + returns (CreateWorkspaceResponse) { + option (google.api.http) = { + post : "/v1/workspace" + body : "*" + }; + } + + + // DeleteWorkspace + // + // Status: ALPHA + // + // Delete a workspace + rpc DeleteWorkspace(DeleteWorkspaceRequest) + returns (DeleteWorkspaceResponse) { + option (google.api.http) = { + delete : "/v1/workspace/{workspace_id}" + body : "*" + }; + } + + // DeleteWorkspace + // + // Status: ALPHA + // + // Claims an anonymous workspace, and transfers the owner to a regular user account. + rpc ClaimWorkspace(ClaimWorkspaceRequest) + returns (ClaimWorkspaceResponse) { + option (google.api.http) = { + post : "/v1/workspace/{workspace_id}/claim" + body : "*" + }; + } + + // MoveWorkspaceData + // + // Status: ALPHA + // + // Claims an anonymous workspace + rpc MoveWorkspaceData(MoveWorkspaceDataRequest) + returns (MoveWorkspaceDataResponse) { + option (google.api.http) = { + post : "/v1/workspace/{workspace_id}/move/{collection_id}" + body : "*" + }; + } +} + // Models: +message CreateWorkspaceRequest {} + +message CreateWorkspaceResponse{ + string workspace_id = 1; + string token = 2; + string access_key = 3; + string secret_key = 4; +} + +message DeleteWorkspaceRequest { + string workspace_id = 1; +} + +message DeleteWorkspaceResponse {} + +message ClaimWorkspaceRequest { + // This can only be called by an registered user, + // that is in possesion of the workspace_id and workspace token + // It will remove the service account and claim all references "as" the user. + string workspace_id = 1; + string token = 2; +} + +message ClaimWorkspaceResponse {} + +message MoveWorkspaceDataRequest { + string workspace_id = 1; + string collection_id = 2; +} + +message MoveWorkspaceDataResponse {} + message CreateNewCollectionRequest { // Collection name string name = 1; From c56a878a527833528364324b1cec30dc49f8c337 Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 16:58:29 +0200 Subject: [PATCH 04/15] feat: First iteration of a hook service --- .../api/hooks/services/v1/hooks_service.proto | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 aruna/api/hooks/services/v1/hooks_service.proto diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto new file mode 100644 index 00000000..5b7490d9 --- /dev/null +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -0,0 +1,69 @@ +syntax = "proto3"; + +package aruna.api.hooks.services.v1; +option go_package = "github.com/ArunaStorage/go-api/aruna/api/storage/services/v1"; +option java_multiple_files = true; +option java_package = "com.github.ArunaStorage.java_api.aruna.api.storage.services.v1"; +option java_outer_classname = "HooksService"; + +import "aruna/api/storage/models/v1/models.proto"; + +// HooksService +// +// Status: ALPHA +// +// A service that enables automatic Hook scheduling +service HooksService { + rpc CreateHook(CreateHookRequest) returns (CreateHookResponse) {} + rpc DeleteHook(DeleteHookRequest) returns (DeleteHookResponse) {} + rpc HookCallback(HookCallbackRequest) returns (HookCallbackResponse) {} +} + +enum TriggerType { + TRIGGER_TYPE_UNSPECIFIED = 0; + TRIGGER_TYPE_OBJECT_CREATED = 1; +} + +message Trigger { + TriggerType trigger_type = 1; + string value = 2; +} + +message Target { + string url = 1; + Credentials credentials = 2; +} + +// Will be updated with additional credential types +message Credentials { + string token = 1; +} + +message CreateHookRequest { + Trigger trigger = 1; + Target target = 2; + string json_template = 3; +} +message CreateHookResponse { + string hook_id = 1; +} + +message DeleteHookRequest { + string hook_id = 1; +} + +message DeleteHookResponse {} + +enum KeyValueType { + KEY_VALUE_TYPE_UNSPECIFIED = 0; + KEY_VALUE_TYPE_HOOK = 1; + KEY_VALUE_TYPE_LABEL = 2; +} + +message HookCallbackRequest { + bool success = 1; + repeated aruna.api.storage.models.v1.KeyValue labels = 2; + repeated aruna.api.storage.models.v1.KeyValue hooks = 3; +} + +message HookCallbackResponse{} \ No newline at end of file From dbf898be5aadb5a04cb5173f82d714bafa3c2b81 Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 17:09:55 +0200 Subject: [PATCH 05/15] feat: Updated hook service --- .../api/hooks/services/v1/hooks_service.proto | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 5b7490d9..100f7cc6 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -15,23 +15,49 @@ import "aruna/api/storage/models/v1/models.proto"; // A service that enables automatic Hook scheduling service HooksService { rpc CreateHook(CreateHookRequest) returns (CreateHookResponse) {} + rpc ListHook(ListHookRequest) returns (ListHookResponse) {} rpc DeleteHook(DeleteHookRequest) returns (DeleteHookResponse) {} rpc HookCallback(HookCallbackRequest) returns (HookCallbackResponse) {} } enum TriggerType { TRIGGER_TYPE_UNSPECIFIED = 0; - TRIGGER_TYPE_OBJECT_CREATED = 1; + TRIGGER_TYPE_HOOK_ADDED = 1; } message Trigger { TriggerType trigger_type = 1; - string value = 2; + string key = 2; + string value = 3; } -message Target { +message ExternalHook { string url = 1; - Credentials credentials = 2; + Credentials credentials = 2; + string json_template = 3; +} + +enum InternalAction { + INTERNAL_ACTION_UNSPECIFIED = 0; + INTERNAL_ACTION_ADD_LABEL = 1; + INTERNAL_ACTION_ADD_HOOK = 2; + INTERNAL_ACTION_CREATE_READ_REFERENCE = 3; + INTERNAL_ACTION_CREATE_WRITE_REFERENCE = 4; +} + +message InternalHook { + InternalAction internal_action = 1; + // Either key or target ID + string target_id = 2; + // Optional value + string value = 3; +} + +message Hook { + oneof hook_type { + ExternalHook external_hook = 1; + InternalHook internal_hook = 2; + } } // Will be updated with additional credential types @@ -41,8 +67,8 @@ message Credentials { message CreateHookRequest { Trigger trigger = 1; - Target target = 2; - string json_template = 3; + Hook hook = 2; + string collection_id = 3; } message CreateHookResponse { string hook_id = 1; @@ -54,16 +80,24 @@ message DeleteHookRequest { message DeleteHookResponse {} -enum KeyValueType { - KEY_VALUE_TYPE_UNSPECIFIED = 0; - KEY_VALUE_TYPE_HOOK = 1; - KEY_VALUE_TYPE_LABEL = 2; -} - message HookCallbackRequest { bool success = 1; repeated aruna.api.storage.models.v1.KeyValue labels = 2; repeated aruna.api.storage.models.v1.KeyValue hooks = 3; } -message HookCallbackResponse{} \ No newline at end of file +message HookCallbackResponse{} + +message ListHookRequest{ + string collection_id = 1; +} + + +message HookInfo { + string hook_id = 1; + Hook hook_info = 2; +} + +message ListHookResponse{ + repeated HookInfo infos = 1; +} \ No newline at end of file From e67a8153c7e9b7cd1fb3b2d7bde26eb7bd32daf5 Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 17:11:01 +0200 Subject: [PATCH 06/15] chore: Update google dependencies --- aruna/api/google | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aruna/api/google b/aruna/api/google index 985b6516..5d285a64 160000 --- a/aruna/api/google +++ b/aruna/api/google @@ -1 +1 @@ -Subproject commit 985b6516d4050697aaeb26e92563eace633aaee2 +Subproject commit 5d285a6449c121ece5ee29305f4856464648fa96 From facfa784e282a7d75a964a827a872bbb472488b5 Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 17:15:23 +0200 Subject: [PATCH 07/15] feat: Added project_name to create_workspace_request --- aruna/api/storage/services/v1/collection_service.proto | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aruna/api/storage/services/v1/collection_service.proto b/aruna/api/storage/services/v1/collection_service.proto index 7a3cb1b2..2776c195 100644 --- a/aruna/api/storage/services/v1/collection_service.proto +++ b/aruna/api/storage/services/v1/collection_service.proto @@ -170,7 +170,9 @@ service WorkspaceService { // Models: -message CreateWorkspaceRequest {} +message CreateWorkspaceRequest { + string project_name = 1; +} message CreateWorkspaceResponse{ string workspace_id = 1; From 61692465f578ac662a9e765737781d82c98928fd Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 17:25:45 +0200 Subject: [PATCH 08/15] feat: Made hooks project specific --- aruna/api/hooks/services/v1/hooks_service.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 100f7cc6..4250ef40 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -68,7 +68,7 @@ message Credentials { message CreateHookRequest { Trigger trigger = 1; Hook hook = 2; - string collection_id = 3; + string project_id = 3; } message CreateHookResponse { string hook_id = 1; @@ -89,7 +89,7 @@ message HookCallbackRequest { message HookCallbackResponse{} message ListHookRequest{ - string collection_id = 1; + string project_id = 1; } From a5b73a78b2abc8875c286e0feb5fe733b6deda9d Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 17:28:20 +0200 Subject: [PATCH 09/15] feat: Added timeout to hook, added Trigger to hookinfo --- aruna/api/hooks/services/v1/hooks_service.proto | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 4250ef40..7249b960 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -68,7 +68,8 @@ message Credentials { message CreateHookRequest { Trigger trigger = 1; Hook hook = 2; - string project_id = 3; + uint64 timeout = 3; + string project_id = 4; } message CreateHookResponse { string hook_id = 1; @@ -96,6 +97,8 @@ message ListHookRequest{ message HookInfo { string hook_id = 1; Hook hook_info = 2; + Trigger trigger = 3; + uint64 timeout = 4; } message ListHookResponse{ From c4bd583a04469abe470291ddb504f516e2a950be Mon Sep 17 00:00:00 2001 From: St4NNi Date: Tue, 20 Jun 2023 17:31:41 +0200 Subject: [PATCH 10/15] feat: Added hooks to build.rs --- tests/build.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/build.rs b/tests/build.rs index 43f61b88..2423281a 100644 --- a/tests/build.rs +++ b/tests/build.rs @@ -46,6 +46,18 @@ fn compile_services() -> Result<(), Box> { protos.push(rel_path); } + let service_entries = fs::read_dir("aruna/api/hooks/services/v1/")?; + + for entry in service_entries { + let dir = entry?; + let rel_path = format!( + "{}{}", + "aruna/api/hooks/services/v1/", + dir.file_name().to_str().unwrap().to_string() + ); + protos.push(rel_path); + } + let service_entries = fs::read_dir("aruna/api/internal/v1/")?; for entry in service_entries { From e5c371d29d48943dec3099a434b672f85bb38ae3 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Wed, 21 Jun 2023 10:57:58 +0200 Subject: [PATCH 11/15] fix: Revert pin collection path Co-authored-by: Jannis Hochmuth --- aruna/api/storage/services/v1/collection_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aruna/api/storage/services/v1/collection_service.proto b/aruna/api/storage/services/v1/collection_service.proto index 2776c195..505ee1be 100644 --- a/aruna/api/storage/services/v1/collection_service.proto +++ b/aruna/api/storage/services/v1/collection_service.proto @@ -87,7 +87,7 @@ service CollectionService { rpc PinCollectionVersion(PinCollectionVersionRequest) returns (PinCollectionVersionResponse) { option (google.api.http) = { - post : "/v1/workspace/" + post : "/v1/collection/{collection_id}/pin" body : "*" }; } From ba3f424b50e624abc731bd68a1832c7daabdacf6 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Wed, 21 Jun 2023 10:58:31 +0200 Subject: [PATCH 12/15] feat: Rename ListHook to ListHooks Co-authored-by: Jannis Hochmuth --- aruna/api/hooks/services/v1/hooks_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 7249b960..406bce59 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -15,7 +15,7 @@ import "aruna/api/storage/models/v1/models.proto"; // A service that enables automatic Hook scheduling service HooksService { rpc CreateHook(CreateHookRequest) returns (CreateHookResponse) {} - rpc ListHook(ListHookRequest) returns (ListHookResponse) {} + rpc ListHooks(ListHooksRequest) returns (ListHooksResponse) {} rpc DeleteHook(DeleteHookRequest) returns (DeleteHookResponse) {} rpc HookCallback(HookCallbackRequest) returns (HookCallbackResponse) {} } From 9fdb404b8032c0d76a46bf1fcbd5d786ee293da3 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Wed, 21 Jun 2023 10:58:47 +0200 Subject: [PATCH 13/15] feat: Rename ListHook to ListHooks Co-authored-by: Jannis Hochmuth --- aruna/api/hooks/services/v1/hooks_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 406bce59..73e1ff26 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -101,6 +101,6 @@ message HookInfo { uint64 timeout = 4; } -message ListHookResponse{ +message ListHooksResponse{ repeated HookInfo infos = 1; } \ No newline at end of file From 8096436ffdb2b47b66ef731a5f3d4b6e382433b4 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Wed, 21 Jun 2023 10:58:56 +0200 Subject: [PATCH 14/15] feat: Rename ListHook to ListHooks Co-authored-by: Jannis Hochmuth --- aruna/api/hooks/services/v1/hooks_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 73e1ff26..18f66a3d 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -89,7 +89,7 @@ message HookCallbackRequest { message HookCallbackResponse{} -message ListHookRequest{ +message ListHooksRequest{ string project_id = 1; } From 4677e343f6defe62614995ba76b3cb4d30ab3d9b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Wed, 21 Jun 2023 10:59:40 +0200 Subject: [PATCH 15/15] feat: Rename hook_info to hook Co-authored-by: Jannis Hochmuth --- aruna/api/hooks/services/v1/hooks_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aruna/api/hooks/services/v1/hooks_service.proto b/aruna/api/hooks/services/v1/hooks_service.proto index 18f66a3d..16a4ca06 100644 --- a/aruna/api/hooks/services/v1/hooks_service.proto +++ b/aruna/api/hooks/services/v1/hooks_service.proto @@ -96,7 +96,7 @@ message ListHooksRequest{ message HookInfo { string hook_id = 1; - Hook hook_info = 2; + Hook hook = 2; Trigger trigger = 3; uint64 timeout = 4; }