Async search: list method and a bunch of various improvements and fixes

api/seqproxyapi/v1/seq_proxy_api.proto

@@ -85,7 +85,8 @@ service SeqProxyApi {
   // Clients should use the search ID returned by StartAsyncSearch.
   rpc FetchAsyncSearchResult(FetchAsyncSearchResultRequest) returns (FetchAsyncSearchResultResponse) {
     option (google.api.http) = {
-      get: "/async-searches/{search_id}"
+      post: "/async-searches/fetch"
+      body: "*"
     };
   }
 
@@ -102,6 +103,14 @@ service SeqProxyApi {
       delete: "/async-searches/{search_id}"
     };
   }
+
+  // Fetch list of async searches.
+  rpc GetAsyncSearchesList(GetAsyncSearchesListRequest) returns (GetAsyncSearchesListResponse) {
+    option (google.api.http) = {
+      post: "/async-searches/list"
+      body: "*"
+    };
+  }
 }
 
 // Custom error code, returned by seq-db proxy.
@@ -281,14 +290,15 @@ enum AsyncSearchStatus {
 
 message FetchAsyncSearchResultResponse {
   AsyncSearchStatus status = 1;
-  ComplexSearchResponse response = 2;
-  google.protobuf.Timestamp started_at = 3;
-  google.protobuf.Timestamp expires_at = 4;
-  optional google.protobuf.Timestamp canceled_at = 5;
+  StartAsyncSearchRequest request = 2;
+  ComplexSearchResponse response = 3;
+  google.protobuf.Timestamp started_at = 4;
+  google.protobuf.Timestamp expires_at = 5;
+  optional google.protobuf.Timestamp canceled_at = 6;
   // Search progress in range [0, 1].
-  double progress = 6;
+  double progress = 7;
   // The size of data stored on disk, in bytes.
-  uint64 disk_usage = 7;
+  uint64 disk_usage = 8;
 }
 
 message CancelAsyncSearchRequest{
@@ -303,6 +313,33 @@ message DeleteAsyncSearchRequest {
 
 message DeleteAsyncSearchResponse {}
 
+message GetAsyncSearchesListRequest {
+  optional AsyncSearchStatus status = 1;
+  // Maximum number of searches to fetch (pagination).
+  int32 size = 2;
+  // Searches offset (pagination).
+  int32 offset = 3;
+  // list of async search ids to filter out result
+  repeated string ids = 4; 
+}
+
+message GetAsyncSearchesListResponse {
+  repeated AsyncSearchesListItem searches = 1;
+}
+
+message AsyncSearchesListItem {
+  string search_id = 1;
+  AsyncSearchStatus status = 2;
+  StartAsyncSearchRequest request = 3;
+  google.protobuf.Timestamp started_at = 4;
+  google.protobuf.Timestamp expires_at = 5;
+  optional google.protobuf.Timestamp canceled_at = 6;
+  // Search progress in range [0, 1].
+  double progress = 7;
+  // The size of data stored on disk, in bytes.
+  uint64 disk_usage = 8;
+}
+
 message GetAggregationRequest {
   SearchQuery query = 1;      // Search query.
   repeated AggQuery aggs = 2; // List of aggregation queries.

api/storeapi/store_api.proto

@@ -20,6 +20,8 @@ service StoreApi {
   rpc CancelAsyncSearch(CancelAsyncSearchRequest) returns (CancelAsyncSearchResponse) {}
 
   rpc DeleteAsyncSearch(DeleteAsyncSearchRequest) returns (DeleteAsyncSearchResponse) {}
+
+  rpc GetAsyncSearchesList(GetAsyncSearchesListRequest) returns (GetAsyncSearchesListResponse) {}
 
   rpc Fetch(FetchRequest) returns (stream BinaryData) {}
 
@@ -176,6 +178,12 @@ message FetchAsyncSearchResultResponse {
 
   repeated AggQuery aggs = 9;
   int64 histogram_interval = 10;
+
+  string query = 11;
+  google.protobuf.Timestamp from = 12;
+  google.protobuf.Timestamp to = 13;
+  google.protobuf.Duration retention = 14;
+  bool with_docs = 15;
 }
 
 message CancelAsyncSearchRequest{
@@ -190,6 +198,35 @@ message DeleteAsyncSearchRequest {
 
 message DeleteAsyncSearchResponse {}
 
+message GetAsyncSearchesListRequest {
+  optional AsyncSearchStatus status = 1;
+  repeated string ids = 2; 
+}
+
+message GetAsyncSearchesListResponse {
+  repeated AsyncSearchesListItem searches = 1;
+}
+
+message AsyncSearchesListItem {
+  string search_id = 1;
+  AsyncSearchStatus status = 2;
+  google.protobuf.Timestamp started_at = 3;
+  google.protobuf.Timestamp expires_at = 4;
+  optional google.protobuf.Timestamp canceled_at = 5;
+  uint64 fracs_done = 6;
+  uint64 fracs_queue = 7;
+  uint64 disk_usage = 8;
+
+  repeated AggQuery aggs = 9;
+  int64 histogram_interval = 10;
+
+  string query = 11;
+  google.protobuf.Timestamp from = 12;
+  google.protobuf.Timestamp to = 13;
+  google.protobuf.Duration retention = 14;
+  bool with_docs = 15;
+}
+
 message IdWithHint {
   string id = 1;
   string hint = 2;

cmd/seq-db/seq-db.go

@@ -289,8 +289,8 @@ func startStore(
 				Async: fracmanager.AsyncSearcherConfig{
 					DataDir:           cfg.AsyncSearch.DataDir,
 					Workers:           cfg.AsyncSearch.Concurrency,
-					MaxSize:           cfg.AsyncSearch.MaxTotalSize,
-					MaxSizePerRequest: cfg.AsyncSearch.MaxSizePerRequest,
+					MaxSize:           int(cfg.AsyncSearch.MaxTotalSize),
+					MaxSizePerRequest: int(cfg.AsyncSearch.MaxSizePerRequest),
 				},
 			},
 			Fetch: storeapi.FetchConfig{

config/config.go

@@ -217,8 +217,8 @@ type Config struct {
 		// By default will be subdirectory in [Config.Storage.DataDir].
 		DataDir           string `config:"data_dir"`
 		Concurrency       int    `config:"concurrency"`
-		MaxTotalSize      int    `config:"max_total_size"`
-		MaxSizePerRequest int    `config:"max_size_per_request"`
+		MaxTotalSize      Bytes  `config:"max_total_size" default:"1GiB"`
+		MaxSizePerRequest Bytes  `config:"max_size_per_request" default:"100MiB"`
 	} `config:"async_search"`
 
 	API struct {