V2.0.0 beta.12

Cargo.toml

@@ -11,7 +11,7 @@ path = "./tests/build.rs"
 
 [dependencies]
 bytes = "1.5.0"
-tonic = "0.10.2"
+tonic = "0.11.0"
 prost = "0.12.1"
-tonic-build = "0.10.2"
+tonic-build = "0.11.0"
 uuid = "1.5.0"

aruna/api/dataproxy/services/v2/bundler_service.proto

@@ -45,6 +45,7 @@ message CreateBundleRequest {
     repeated string resource_ids = 1;
     string filename = 2; // .tar.gz / .zip
     google.protobuf.Timestamp expires_at = 3; // Default 1 Month
+    bool once = 4; // Default false (expires after first download)
 }
 
 message CreateBundleResponse {

aruna/api/dataproxy/services/v2/dataproxy_service.proto

@@ -8,6 +8,7 @@ option java_outer_classname = "DataProxyService";
 
 import "google/api/annotations.proto";
 import "google/api/visibility.proto";
+import "aruna/api/storage/models/v2/models.proto";
 
 // DataproxyService
 //
@@ -26,7 +27,7 @@ service DataproxyReplicationService {
 
     // InitReplication
     //
-    // Status: ALPHA
+    // Status: UNIMPLEMENTED
     //
     // Provides the necessary url to init replication
     rpc PushReplication(PushReplicationRequest) returns (PushReplicationResponse) {}
@@ -56,15 +57,42 @@ service DataproxyUserService {
     // Authorized method that needs a aruna-token to exchange for dataproxy
     // specific S3AccessKey and S3SecretKey
     rpc GetCredentials(GetCredentialsRequest) returns (GetCredentialsResponse) {
+        option (google.api.http) = {
+            get : "/v2/credentials"
+        };
+    }
+
+    // CreateOrUpdateCredentials
+    //
+    // Status: BETA
+    //
+    // Authorized method that needs a aruna-token to exchange for dataproxy
+    // specific S3AccessKey and S3SecretKey
+    rpc CreateOrUpdateCredentials(CreateOrUpdateCredentialsRequest) returns (CreateOrUpdateCredentialsResponse) {
         option (google.api.http) = {
             post : "/v2/credentials"
             body : "*"
         };
     }
-    // PushReplica
+
+
+    // RevokeCredentials
     //
     // Status: BETA
     //
+    // Authorized method that needs a aruna-token
+    // Revokes the current credentials
+    rpc RevokeCredentials(RevokeCredentialsRequest) returns (RevokeCredentialsResponse) {
+        option (google.api.http) = {
+            delete : "/v2/credentials"
+        };
+    }
+
+
+    // PushReplica
+    //
+    // Status: UNIMPLEMENTED
+    //
     // Manually transfers a replica to another data-proxy
     rpc PushReplica(PushReplicaRequest) returns (PushReplicaResponse) {
         option (google.api.http) = {
@@ -74,7 +102,7 @@ service DataproxyUserService {
     }
     // PullReplica
     //
-    // Status: BETA
+    // Status: UNIMPLEMENTED
     //
     // Manually request data to be transferred to this data-proxy
     rpc PullReplica(PullReplicaRequest) returns (PullReplicaResponse) {
@@ -83,9 +111,10 @@ service DataproxyUserService {
             body : "*"
         };
     }
-    // PullReplica
+
+    // ReplicationStatus
     //
-    // Status: BETA
+    // Status: UNIMPLEMENTED
     //
     // Status of the previous replication request
     rpc ReplicationStatus(ReplicationStatusRequest) returns (ReplicationStatusResponse) {
@@ -95,6 +124,16 @@ service DataproxyUserService {
     }
 }
 
+service DataproxyIngestionService {
+    option (google.api.api_visibility).restriction = "PROXY";
+    // IngestExistingObject
+    //
+    // Status: ALPHA
+    //
+    // Ingest an existing object into backend
+    rpc IngestExistingObject(IngestExistingObjectRequest) returns (IngestExistingObjectResponse) {}
+}
+
 
 // ----- PullReplication -----
 // PROXY A (data) <--> PROXY B (wants data)
@@ -143,9 +182,9 @@ message PullReplicationRequest {
 // Messages (responses) from PROXY A
 message ObjectInfo {
     string object_id = 1;
-    int64 chunks = 2;
+    int64 chunks = 2; // == (Compressed_size / (65536 + 28)) + 1
     int64 raw_size  = 3;
-    repeated uint32 block_list = 4;
+    int64 compressed_size = 4;
     optional string extra = 5; // JSON encoded proxy specific extra fields
 }
 
@@ -192,6 +231,18 @@ message GetCredentialsResponse {
     string secret_key = 2;
 }
 
+message CreateOrUpdateCredentialsRequest {
+}
+
+message CreateOrUpdateCredentialsResponse {
+    string access_key = 1;
+    string secret_key = 2;
+}
+
+message RevokeCredentialsRequest {}
+
+message RevokeCredentialsResponse {}
+
 message S3Path {
     string bucket = 1;
     string key = 2;
@@ -326,4 +377,43 @@ message InitLocationResponse {
     ObjectLocation location = 1;
 }
 
+message IngestResource {
+  // object name
+  string name = 1;
+  // title
+  string title = 2;
+  // description
+  string description = 3;
+  // Authors
+  repeated storage.models.v2.Author authors = 4;
+  // object specific labels / hooks
+  repeated storage.models.v2.KeyValue key_values = 5;
+  // Internal / External relations (URLs / IDs from external sources)
+  repeated storage.models.v2.Relation relations = 6;
+  // DataClass
+  storage.models.v2.DataClass data_class = 7;
+  // Ignored if Collection | Dataset
+  repeated storage.models.v2.Hash hashes = 8;
+  string metadata_license_tag = 9;
+  string data_license_tag = 10;
+}
+
+message IngestExistingObjectRequest {
+    string project_id = 1;
+    oneof collection {
+        string collection_id = 2;
+        IngestResource collection_resource = 3;
+    }
+    oneof dataset {
+        string dataset_id = 4;
+        IngestResource dataset_resource = 5;
+    }
+    IngestResource object = 6;
+    string path = 7; // "s3://bucket/key" or "file:///foo/bar/baz.txt" must be a valid file
+}
+
+message IngestExistingObjectResponse {
+    string object_id = 1;
+}
+
 

aruna/api/storage/models/v2/models.proto

@@ -141,6 +141,9 @@ message User {
   string email = 5;
   // User attributes
   UserAttributes attributes = 6;
+  // First and Last name
+  string first_name = 7;
+  string last_name = 8;
 }
 
 message Permission {
@@ -168,11 +171,18 @@ message Pubkey {
   string location = 3;
 }
 
-message CustomAttributes {
+message CustomAttribute {
   string attribute_name = 1;
   string attribute_value = 2;
 }
 
+message DataProxyAttribute {
+  string attribute_name = 1;
+  string attribute_value = 2;
+  string signature = 3;
+  string proxy_id = 4;
+}
+
 message OidcMapping {
   string external_id = 1;
   string oidc_url = 2;
@@ -183,9 +193,11 @@ message UserAttributes {
   bool service_account = 2;
   repeated Token tokens = 3;
   repeated string trusted_endpoints = 4;
-  repeated CustomAttributes custom_attributes = 5; 
+  repeated CustomAttribute custom_attributes = 5; 
   repeated Permission personal_permissions = 6;
   repeated OidcMapping external_ids = 7;
+  string pubkey = 8;
+  repeated DataProxyAttribute data_proxy_attributes = 9;
 }
 
 // --------------- RELATION / KEYVALUES -------------------
@@ -289,6 +301,19 @@ message License {
   string url = 4; // https://creativecommons.org/licenses/by-sa/4.0/
 }
 
+message Author {
+  string first_name = 1;
+  string last_name = 2;
+  optional string email = 3;
+  optional string orcid = 4;
+  optional string id = 5;
+}
+
+message RuleBinding {
+  string rule_id = 1;
+  string origin = 2;
+}
+
 // ------ Resources ----------
 
 message GenericResource {
@@ -303,6 +328,7 @@ message GenericResource {
 message Project {
   string id = 1;
   string name = 2; // Short name according to BucketNamingRules
+  string title = 15; // Long name
   string description = 3; // Long name
   // Project specific labels / hooks
   repeated KeyValue key_values = 4;
@@ -312,17 +338,20 @@ message Project {
   DataClass data_class = 7;
   google.protobuf.Timestamp created_at = 8;
   string created_by = 9;
+  repeated Author authors = 16;
   Status status = 10;
   bool dynamic = 11;
   repeated DataEndpoint endpoints = 12;
   string metadata_license_tag = 13;
   string default_data_license_tag = 14;
+  repeated RuleBinding rule_bindings = 17;
 }
 
 
 message Collection {
   string id = 1; // ASDASDASDOPASKIDPO
   string name = 2; // my_mags
+  string title = 15;
   string description = 3; // ENA asda234928349028 MAG 1293819203819028i V1
   // Collection specific labels / hooks
   repeated KeyValue key_values = 4;
@@ -332,16 +361,19 @@ message Collection {
   DataClass data_class = 7;
   google.protobuf.Timestamp created_at = 8;
   string created_by = 9;
+  repeated Author authors = 16;
   Status status = 10;
   bool dynamic = 11;
   repeated DataEndpoint endpoints = 12;
   string metadata_license_tag = 13;
   string default_data_license_tag = 14;
+  repeated RuleBinding rule_bindings = 17;
 }
 
 message Dataset {
   string id = 1;
   string name = 2;
+  string title = 15; // Long name
   string description = 3;
   // Dataset specific labels / hooks
   repeated KeyValue key_values = 4;
@@ -351,16 +383,19 @@ message Dataset {
   DataClass data_class = 7;
   google.protobuf.Timestamp created_at = 8;
   string created_by = 9;
+  repeated Author authors = 16;
   Status status = 10;
   bool dynamic = 11;
   repeated DataEndpoint endpoints = 12;
   string metadata_license_tag = 13;
   string default_data_license_tag = 14;
+  repeated RuleBinding rule_bindings = 17;
 }
 
 message Object {
   string id = 1;
   string name = 2;
+  string title = 16; // Long name
   string description = 3;
   // Collection specific labels / hooks
   repeated KeyValue key_values = 4;
@@ -370,11 +405,13 @@ message Object {
   DataClass data_class = 7;
   google.protobuf.Timestamp created_at = 8;
   string created_by = 9;
+  repeated Author authors = 17;
   Status status = 10;
   bool dynamic = 11;
   repeated DataEndpoint endpoints = 12;
   // Object specific attributes
   repeated Hash hashes = 13;  
   string metadata_license_tag = 14;
   string data_license_tag = 15;
+  repeated RuleBinding rule_bindings = 18;
 }