METRON-1114: Add group by capabilities to search REST endpoint

metron-interface/metron-rest/README.md

@@ -199,10 +199,11 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
 | [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopicname)|
 | [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopicname)|
 | [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopicnamesample)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
+| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
+| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
 | [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadata)|
-| [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadatacommon)|
+| [ `GET /api/v1/search/column/metadata`](#get-apiv1searchcolumnmetadata)|
+| [ `GET /api/v1/search/column/metadata/common`](#get-apiv1searchcolumnmetadatacommon)|
 | [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)|
 | [ `GET /api/v1/sensor/enrichment/config/list/available/enrichments`](#get-apiv1sensorenrichmentconfiglistavailableenrichments)|
 | [ `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`](#get-apiv1sensorenrichmentconfiglistavailablethreattriageaggregators)|
@@ -353,6 +354,24 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
     * 200 - Returns sample message
     * 404 - Either Kafka topic is missing or contains no messages
 
+### `POST /api/v1/search/search`
+  * Description: Searches the indexing store
+  * Input:
+      * searchRequest - Search request
+  * Returns:
+    * 200 - Search response
+
+### `POST /api/v1/search/group`
+  * Description: Searches the indexing store and returns field groups. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending.  A groupOrder type of count will sort based on then number of documents in a group while a groupType of term will sort by the groupBy term.
+  * Input:
+      * groupRequest - Group request
+        * indices - list of indices to search
+        * query - lucene query
+        * scoreField - field used to compute a total score for each group
+        * groups - List of groups (field name and sort order) 
+  * Returns:
+    * 200 - Group response
+
 ### `GET /api/v1/search/findOne`
   * Description: Returns latest document for a guid and sensor
   * Input:
@@ -369,13 +388,6 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
   * Returns:
     * 200 - Document representing the output
     * 404 - Document with UUID and sensor type not found
-
-### `GET /api/v1/search/search`
-  * Description: Searches the indexing store
-  * Input:
-      * searchRequest - Search request
-  * Returns:
-    * 200 - Search results
 
 ### `GET /api/v1/search/column/metadata`
   * Description: Get column metadata for each index in the list of indicies

metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java

@@ -56,6 +56,7 @@ public class MetronRestConstants {
   public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab";
 
   public static final String SEARCH_MAX_RESULTS = "search.max.results";
+  public static final String SEARCH_MAX_GROUPS = "search.max.groups";
   public static final String INDEX_DAO_IMPL = "index.dao.impl";
   public static final String INDEX_HBASE_TABLE_PROVIDER_IMPL = "index.hbase.provider";
 }

metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java

@@ -17,6 +17,8 @@
  */
 package org.apache.metron.rest.config;
 
+import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL;
+
 import org.apache.metron.hbase.HTableProvider;
 import org.apache.metron.hbase.TableProvider;
 import org.apache.metron.indexing.dao.AccessConfig;
@@ -28,14 +30,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Profile;
 import org.springframework.core.env.Environment;
 
-import java.lang.reflect.InvocationTargetException;
-
-import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-
 @Configuration
 public class IndexConfig {
 
@@ -55,9 +51,11 @@ public IndexDao indexDao() {
     try {
       String hbaseProviderImpl = environment.getProperty(MetronRestConstants.INDEX_HBASE_TABLE_PROVIDER_IMPL, String.class, null);
       String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null);
-      int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, -1);
+      int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, 1000);
+      int searchMaxGroups = environment.getProperty(MetronRestConstants.SEARCH_MAX_GROUPS, Integer.class, 1000);
       AccessConfig config = new AccessConfig();
       config.setMaxSearchResults(searchMaxResults);
+      config.setMaxSearchGroups(searchMaxGroups);
       config.setGlobalConfigSupplier(() -> {
         try {
           return globalConfigService.get();

metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java

@@ -21,7 +21,8 @@
 import io.swagger.annotations.ApiParam;
 import io.swagger.annotations.ApiResponse;
 import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
 import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -38,7 +39,6 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.List;
-import java.util.Map;
 
 @RestController
 @RequestMapping("/api/v1/search")
@@ -54,6 +54,15 @@ ResponseEntity<SearchResponse> search(final @ApiParam(name = "searchRequest", va
     return new ResponseEntity<>(searchService.search(searchRequest), HttpStatus.OK);
   }
 
+  @ApiOperation(value = "Searches the indexing store and returns field groups. "
+      + "Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. "
+      + "The default sorting within groups is by count descending.")
+  @ApiResponse(message = "Group response", code = 200)
+  @RequestMapping(value = "/group", method = RequestMethod.POST)
+  ResponseEntity<GroupResponse> group(final @ApiParam(name = "groupRequest", value = "Group request", required = true) @RequestBody GroupRequest groupRequest) throws RestException {
+    return new ResponseEntity<>(searchService.group(groupRequest), HttpStatus.OK);
+  }
+
   @ApiOperation(value = "Returns latest document for a guid and sensor")
   @ApiResponse(message = "Document representing the output", code = 200)
   @RequestMapping(value = "/findOne", method = RequestMethod.POST)

metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java

@@ -18,7 +18,8 @@
 package org.apache.metron.rest.service;
 
 import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
 import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.indexing.dao.search.SearchRequest;
@@ -27,11 +28,11 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.List;
-import java.util.Map;
 
 public interface SearchService {
 
   SearchResponse search(SearchRequest searchRequest) throws RestException;
+  GroupResponse group(GroupRequest groupRequest) throws RestException;
   Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException;
   Map<String, Map<String, FieldType>> getColumnMetadata(List<String> indices) throws RestException;
   Map<String, FieldType> getCommonColumnMetadata(List<String> indices) throws RestException;

metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java

@@ -19,10 +19,11 @@
 
 import org.apache.metron.indexing.dao.IndexDao;
 import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
 import org.apache.metron.indexing.dao.search.InvalidSearchException;
 import org.apache.metron.indexing.dao.search.SearchRequest;
 import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.update.Document;
 import org.apache.metron.indexing.dao.search.FieldType;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.service.SearchService;
@@ -34,7 +35,6 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.List;
-import java.util.Map;
 
 @Service
 public class SearchServiceImpl implements SearchService {
@@ -57,6 +57,16 @@ public SearchResponse search(SearchRequest searchRequest) throws RestException {
     }
   }
 
+  @Override
+  public GroupResponse group(GroupRequest groupRequest) throws RestException {
+    try {
+      return dao.group(groupRequest);
+    }
+    catch(InvalidSearchException ise) {
+      throw new RestException(ise.getMessage(), ise);
+    }
+  }
+
   @Override
   public Optional<Map<String, Object>> getLatest(GetRequest request) throws RestException {
     try {

metron-interface/metron-rest/src/main/resources/application.yml

@@ -44,6 +44,7 @@ curator:
 search:
   max:
     results: 1000
+    groups: 1000
 
 index:
   dao:

metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java

@@ -24,7 +24,6 @@
 import org.json.simple.parser.ParseException;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -44,7 +43,6 @@
 import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
 import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
 import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
@@ -174,6 +172,22 @@ public void test() throws Exception {
             .andExpect(jsonPath("$.responseCode").value(500))
             .andExpect(jsonPath("$.message").value("Search result size must be less than 100"));
 
+    this.mockMvc.perform(post(searchUrl + "/group").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.groupByQuery))
+            .andExpect(status().isOk())
+            .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+            .andExpect(jsonPath("$.*", hasSize(2)))
+            .andExpect(jsonPath("$.groupedBy").value("is_alert"))
+            .andExpect(jsonPath("$.groupResults.*", hasSize(1)))
+            .andExpect(jsonPath("$.groupResults[0].*", hasSize(5)))
+            .andExpect(jsonPath("$.groupResults[0].key").value("is_alert_value"))
+            .andExpect(jsonPath("$.groupResults[0].total").value(10))
+            .andExpect(jsonPath("$.groupResults[0].groupedBy").value("latitude"))
+            .andExpect(jsonPath("$.groupResults[0].groupResults.*", hasSize(1)))
+            .andExpect(jsonPath("$.groupResults[0].groupResults[0].*", hasSize(3)))
+            .andExpect(jsonPath("$.groupResults[0].groupResults[0].key").value("latitude_value"))
+            .andExpect(jsonPath("$.groupResults[0].groupResults[0].total").value(10))
+            .andExpect(jsonPath("$.groupResults[0].groupResults[0].score").value(50));
+
     this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]"))
             .andExpect(status().isOk())
             .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))