From 3650ef9aeea49d8238bf2e409ec0885120722b59 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 17 Aug 2017 09:36:05 -0500 Subject: [PATCH 1/7] initial commit --- .../SearchControllerIntegrationTest.java | 13 +- .../elasticsearch/dao/ElasticsearchDao.java | 213 +++++++++++------- .../indexing/dao/search/SearchRequest.java | 9 + .../indexing/dao/search/SearchResponse.java | 24 +- .../dao/search/SearchResultGroup.java | 75 ++++++ .../metron/indexing/dao/InMemoryDao.java | 14 +- .../indexing/dao/SearchIntegrationTest.java | 141 ++++++++++++ 7 files changed, 400 insertions(+), 89 deletions(-) create mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index e75c356cee..2e2d8f2fb7 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -24,7 +24,6 @@ import org.json.simple.parser.ParseException; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -44,7 +43,6 @@ import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -174,6 +172,17 @@ public void test() throws Exception { .andExpect(jsonPath("$.responseCode").value(500)) .andExpect(jsonPath("$.message").value("Search result size must be less than 100")); + this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.groupByQuery)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(3))) + .andExpect(jsonPath("$.total").value(10)) + .andExpect(jsonPath("$.groupedBy").value("groupByField")) + .andExpect(jsonPath("$.groups.*", hasSize(1))) + .andExpect(jsonPath("$.groups[0].key").value("groupByValue")) + .andExpect(jsonPath("$.groups[0].total").value(10)) + .andExpect(jsonPath("$.groups[0].results.*", hasSize(10))); + this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 217da84853..08aeb39995 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -19,58 +19,52 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.writer.WriterConfiguration; -import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; import org.apache.metron.indexing.dao.AccessConfig; -import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.IndexDao; -import org.apache.metron.indexing.dao.search.*; +import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.*; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; +import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.search.SearchResultGroup; +import org.apache.metron.indexing.dao.update.Document; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.mapper.ip.IpFieldMapper; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; -import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; -import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; +import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.*; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; public class ElasticsearchDao implements IndexDao { private transient TransportClient client; @@ -110,24 +104,21 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx if (searchRequest.getSize() > accessConfig.getMaxSearchResults()) { throw new InvalidSearchException("Search result size must be less than " + accessConfig.getMaxSearchResults()); } - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .size(searchRequest.getSize()) - .from(searchRequest.getFrom()) - .query(new QueryStringQueryBuilder(searchRequest.getQuery())) - .fetchSource(true) - .trackScores(true); - for (SortField sortField : searchRequest.getSort()) { - FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(sortField.getField()); - if (sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC) { - fieldSortBuilder.order(org.elasticsearch.search.sort.SortOrder.DESC); - } else { - fieldSortBuilder.order(org.elasticsearch.search.sort.SortOrder.ASC); - } - searchSourceBuilder = searchSourceBuilder.sort(fieldSortBuilder); + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(new QueryStringQueryBuilder(searchRequest.getQuery())); + Optional> groupByFields = searchRequest.getGroupByFields(); + if (groupByFields.isPresent()) { + searchSourceBuilder.aggregation(getGroupsTermBuilder(searchRequest, 0)); + } else { + searchSourceBuilder.size(searchRequest.getSize()) + .from(searchRequest.getFrom()) + .fetchSource(true) + .trackScores(true); + searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); } Optional> facetFields = searchRequest.getFacetFields(); if (facetFields.isPresent()) { - addFacetFields(searchSourceBuilder, facetFields.get()); + facetFields.get().forEach(field -> searchSourceBuilder.aggregation(new TermsBuilder(getFacentAggregationName(field)).field(field))); } String[] wildcardIndices = searchRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[searchRequest.getIndices().size()]); org.elasticsearch.action.search.SearchResponse elasticsearchResponse; @@ -139,23 +130,25 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx } SearchResponse searchResponse = new SearchResponse(); searchResponse.setTotal(elasticsearchResponse.getHits().getTotalHits()); - searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> { - SearchResult searchResult = new SearchResult(); - searchResult.setId(searchHit.getId()); - searchResult.setSource(searchHit.getSource()); - searchResult.setScore(searchHit.getScore()); - searchResult.setIndex(searchHit.getIndex()); - return searchResult; - }).collect(Collectors.toList())); - if (facetFields.isPresent()) { + if (!groupByFields.isPresent()) { + searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(this::getSearchResult).collect(Collectors.toList())); + } + if (groupByFields.isPresent() || facetFields.isPresent()) { Map commonColumnMetadata; try { commonColumnMetadata = getCommonColumnMetadata(searchRequest.getIndices()); } catch (IOException e) { throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(searchRequest.getIndices().toArray()))); } - searchResponse.setFacetCounts(getFacetCounts(facetFields.get(), elasticsearchResponse.getAggregations(), commonColumnMetadata )); + if (groupByFields.isPresent()) { + searchResponse.setGroupedBy(groupByFields.get().get(0)); + searchResponse.setGroups(getGroups(groupByFields.get(), 0, elasticsearchResponse.getAggregations(), commonColumnMetadata)); + } + if (facetFields.isPresent()) { + searchResponse.setFacetCounts(getFacetCounts(facetFields.get(), elasticsearchResponse.getAggregations(), commonColumnMetadata )); + } } + return searchResponse; } @@ -314,43 +307,93 @@ protected String[] getLatestIndices(List includeIndices) { return latestIndices.values().toArray(new String[latestIndices.size()]); } - public void addFacetFields(SearchSourceBuilder searchSourceBuilder, List fields) { - for(String field: fields) { - searchSourceBuilder = searchSourceBuilder.aggregation(new TermsBuilder(getAggregationName(field)).field(field)); - } + private org.elasticsearch.search.sort.SortOrder getElasticsearchSortOrder( + org.apache.metron.indexing.dao.search.SortOrder sortOrder) { + return sortOrder == org.apache.metron.indexing.dao.search.SortOrder.DESC ? + org.elasticsearch.search.sort.SortOrder.DESC : org.elasticsearch.search.sort.SortOrder.ASC; } public Map> getFacetCounts(List fields, Aggregations aggregations, Map commonColumnMetadata) { Map> fieldCounts = new HashMap<>(); for (String field: fields) { Map valueCounts = new HashMap<>(); - Aggregation aggregation = aggregations.get(getAggregationName(field)); - if (aggregation instanceof LongTerms) { - LongTerms longTerms = (LongTerms) aggregation; - FieldType type = commonColumnMetadata.get(field); - if (FieldType.IP.equals(type)) { - longTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(IpFieldMapper.longToIp((Long) bucket.getKey()), bucket.getDocCount())); - } else if (FieldType.BOOLEAN.equals(type)) { - longTerms.getBuckets().stream().forEach(bucket -> { - String key = (Long) bucket.getKey() == 1 ? "true" : "false"; - valueCounts.put(key, bucket.getDocCount()); - }); - } else { - longTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(bucket.getKeyAsString(), bucket.getDocCount())); - } - } else if (aggregation instanceof DoubleTerms) { - DoubleTerms doubleTerms = (DoubleTerms) aggregation; - doubleTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(bucket.getKeyAsString(), bucket.getDocCount())); - } else if (aggregation instanceof StringTerms) { - StringTerms stringTerms = (StringTerms) aggregation; - stringTerms.getBuckets().stream().forEach(bucket -> valueCounts.put(bucket.getKeyAsString(), bucket.getDocCount())); + Aggregation aggregation = aggregations.get(getFacentAggregationName(field)); + if (aggregation instanceof Terms) { + Terms terms = (Terms) aggregation; + terms.getBuckets().stream().forEach(bucket -> valueCounts.put(formatKey(bucket.getKey(), commonColumnMetadata.get(field)), bucket.getDocCount())); } fieldCounts.put(field, valueCounts); } return fieldCounts; } - private String getAggregationName(String field) { + private String formatKey(Object key, FieldType type) { + if (FieldType.IP.equals(type)) { + return IpFieldMapper.longToIp((Long) key); + } else if (FieldType.BOOLEAN.equals(type)) { + return (Long) key == 1 ? "true" : "false"; + } else { + return key.toString(); + } + } + + private TermsBuilder getGroupsTermBuilder(SearchRequest searchRequest, int index) { + List fields = searchRequest.getGroupByFields().get(); + String field = fields.get(index); + String aggregationName = getGroupByAggregationName(field); + if (index < fields.size() - 1) { + return new TermsBuilder(aggregationName).field(field) + .subAggregation(getGroupsTermBuilder(searchRequest, index + 1)); + } else { + TopHitsBuilder topHitsBuilder = new TopHitsBuilder("top_hits").setSize(searchRequest.getSize()) + .setFetchSource(true); + searchRequest.getSort().forEach(sortField -> + topHitsBuilder.addSort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); + return new TermsBuilder(aggregationName).field(field).subAggregation(topHitsBuilder); + } + } + + private List getGroups(List fields, int index, Aggregations aggregations, Map commonColumnMetadata) { + String field = fields.get(index); + Terms terms = aggregations.get(getGroupByAggregationName(field)); + List searchResultGroups = new ArrayList<>(); + if (index < fields.size() - 1) { + String childField = fields.get(index + 1); + for(Bucket bucket: terms.getBuckets()) { + SearchResultGroup searchResultGroup = new SearchResultGroup(); + searchResultGroup.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); + searchResultGroup.setTotal(bucket.getDocCount()); + searchResultGroup.setGroupedBy(childField); + searchResultGroup.setGroups(getGroups(fields, index + 1, bucket.getAggregations(), commonColumnMetadata)); + searchResultGroups.add(searchResultGroup); + } + } else { + for(Bucket bucket: terms.getBuckets()) { + SearchResultGroup searchResultGroup = new SearchResultGroup(); + searchResultGroup.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); + searchResultGroup.setTotal(bucket.getDocCount()); + TopHits topHits = bucket.getAggregations().get("top_hits"); + searchResultGroup.setResults(Arrays.stream(topHits.getHits().getHits()).map(this::getSearchResult).collect(Collectors.toList())); + searchResultGroups.add(searchResultGroup); + } + } + return searchResultGroups; + } + + private SearchResult getSearchResult(SearchHit searchHit) { + SearchResult searchResult = new SearchResult(); + searchResult.setId(searchHit.getId()); + searchResult.setSource(searchHit.getSource()); + searchResult.setScore(searchHit.getScore()); + searchResult.setIndex(searchHit.getIndex()); + return searchResult; + } + + private String getFacentAggregationName(String field) { return String.format("%s_count", field); } + + private String getGroupByAggregationName(String field) { + return String.format("%s_group", field); + } } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java index 897f918fe3..eadd31daa1 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java @@ -28,6 +28,7 @@ public class SearchRequest { private int size; private int from; private List sort; + private List groupByFields; private List facetFields; public SearchRequest() { @@ -99,6 +100,14 @@ public void setSort(List sort) { this.sort = sort; } + public Optional> getGroupByFields() { + return groupByFields == null || groupByFields.size() == 0 ? Optional.empty() : Optional.of(groupByFields); + } + + public void setGroupByFields(List groupByFields) { + this.groupByFields = groupByFields; + } + public Optional> getFacetFields() { return facetFields == null || facetFields.size() == 0 ? Optional.empty() : Optional.of(facetFields); } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java index aad489a1e9..88f16206bf 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java @@ -26,7 +26,9 @@ public class SearchResponse { private long total; - private List results = new ArrayList<>(); + private List results; + private String groupedBy; + private List groups; private Map> facetCounts; /** @@ -45,6 +47,7 @@ public void setTotal(long total) { * The list of results * @return */ + @JsonInclude(JsonInclude.Include.NON_NULL) public List getResults() { return results; } @@ -53,6 +56,25 @@ public void setResults(List results) { this.results = results; } + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getGroupedBy() { + return groupedBy; + } + + public void setGroupedBy(String groupedBy) { + this.groupedBy = groupedBy; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getGroups() { + return groups; + } + + public void setGroups( + List groups) { + this.groups = groups; + } + @JsonInclude(JsonInclude.Include.NON_NULL) public Map> getFacetCounts() { return facetCounts; diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java new file mode 100644 index 0000000000..6a74e4da76 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.indexing.dao.search; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.ArrayList; +import java.util.List; + +public class SearchResultGroup { + + private String key; + private long total; + private List results; + private String groupedBy; + private List groups; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public long getTotal() { + return total; + } + + public void setTotal(long total) { + this.total = total; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getResults() { + return results; + } + + public void setResults(List results) { + this.results = results; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getGroupedBy() { + return groupedBy; + } + + public void setGroupedBy(String groupedBy) { + this.groupedBy = groupedBy; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getGroups() { + return groups; + } + + public void setGroups(List groups) { + this.groups = groups; + } +} diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java index 2d146e0a8e..0dfaab56ca 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -21,6 +21,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Iterables; +import java.util.stream.Collectors; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.utils.JSONUtils; @@ -73,7 +74,18 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx finalResp.add(response.get(i)); } ret.setTotal(response.size()); - ret.setResults(finalResp); + Optional> groupByFields = searchRequest.getGroupByFields(); + if (groupByFields.isPresent()) { + ret.setGroupedBy("groupByField"); + SearchResultGroup searchResultGroup = new SearchResultGroup(); + searchResultGroup.setTotal(response.size()); + searchResultGroup.setKey("groupByValue"); + searchResultGroup.setResults(finalResp); + ret.setGroups(Arrays.asList(searchResultGroup)); + } else { + ret.setResults(finalResp); + } + return ret; } diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index e2622698a0..df679ff17c 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -24,6 +24,7 @@ import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; +import org.apache.metron.indexing.dao.search.SearchResultGroup; import org.apache.metron.integration.InMemoryComponent; import org.junit.*; @@ -213,6 +214,42 @@ public abstract class SearchIntegrationTest { @Multiline public static String exceededMaxResultsQuery; + /** + * { + * "groupByFields":["is_alert","latitude"], + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String groupByQuery; + + /** + * { + * "groupByFields":["is_alert"], + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 3, + * "sort": [ + * { + * "field": "ip_src_port", + * "sortOrder": "asc" + * } + * ] + * } + */ + @Multiline + public static String sortedSizeGroupByQuery; + protected static IndexDao dao; protected static InMemoryComponent indexComponent; @@ -227,6 +264,106 @@ public synchronized void setup() throws Exception { @Test public void test() throws Exception { + { + SearchRequest request = JSONUtils.INSTANCE.load(sortedSizeGroupByQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + Assert.assertNull(response.getResults()); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List isAlertGroups = response.getGroups(); + Assert.assertEquals(2, isAlertGroups.size()); + Collections.sort(isAlertGroups, this::compareGroups); + + // isAlert == false group + SearchResultGroup falseGroup = isAlertGroups.get(0); + Assert.assertNull(falseGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getGroups()); + Assert.assertEquals(4, falseGroup.getTotal()); + List falseGroupResults = falseGroup.getResults(); + Assert.assertEquals(3, falseGroupResults.size()); + Assert.assertEquals(8001, falseGroupResults.get(0).getSource().get("ip_src_port")); + Assert.assertEquals(8003, falseGroupResults.get(1).getSource().get("ip_src_port")); + Assert.assertEquals(8005, falseGroupResults.get(2).getSource().get("ip_src_port")); + + // isAlert == true group + SearchResultGroup trueGroup = isAlertGroups.get(1); + Assert.assertNull(trueGroup.getGroupedBy()); + Assert.assertNull(trueGroup.getGroups()); + Assert.assertEquals(6, trueGroup.getTotal()); + List trueGroupResults = trueGroup.getResults(); + Assert.assertEquals(3, trueGroupResults.size()); + Assert.assertEquals(8002, trueGroupResults.get(0).getSource().get("ip_src_port")); + Assert.assertEquals(8004, trueGroupResults.get(1).getSource().get("ip_src_port")); + Assert.assertEquals(8006, trueGroupResults.get(2).getSource().get("ip_src_port")); + } + + { + SearchRequest request = JSONUtils.INSTANCE.load(groupByQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + Assert.assertNull(response.getResults()); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List isAlertGroups = response.getGroups(); + Assert.assertEquals(2, isAlertGroups.size()); + Collections.sort(isAlertGroups, this::compareGroups); + + // isAlert == false group + SearchResultGroup falseGroup = isAlertGroups.get(0); + Assert.assertEquals("false", falseGroup.getKey()); + Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getResults()); + List falseLatitudeGroups = falseGroup.getGroups(); + Assert.assertEquals(2, falseLatitudeGroups.size()); + Collections.sort(falseLatitudeGroups, this::compareGroups); + + // isAlert == false && latitude == 48.0001 group + SearchResultGroup falseLatitudeGroup1 = falseLatitudeGroups.get(0); + Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); + List falseLatitudeGroup1Results = falseLatitudeGroup1.getResults(); + Assert.assertEquals(1, falseLatitudeGroup1Results.size()); + Assert.assertEquals("192.168.1.2", falseLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); + + // isAlert == false && latitude == 48.5839 group + SearchResultGroup falseLatitudeGroup2 = falseLatitudeGroups.get(1); + Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + List falseLatitudeGroup2Results = falseLatitudeGroup2.getResults(); + Assert.assertEquals(3, falseLatitudeGroup2Results.size()); + Assert.assertEquals("192.168.1.8", falseLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.7", falseLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.6", falseLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); + + // isAlert == true group + SearchResultGroup trueGroup = isAlertGroups.get(1); + Assert.assertEquals("true", trueGroup.getKey()); + Assert.assertEquals("latitude", trueGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getResults()); + List trueLatitudeGroups = trueGroup.getGroups(); + Assert.assertEquals(2, trueLatitudeGroups.size()); + Collections.sort(trueLatitudeGroups, this::compareGroups); + + // isAlert == true && latitude == 48.0001 group + SearchResultGroup trueLatitudeGroup1 = trueLatitudeGroups.get(0); + Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); + List trueLatitudeGroup1Results = trueLatitudeGroup1.getResults(); + Assert.assertEquals(1, trueLatitudeGroup1Results.size()); + Assert.assertEquals("192.168.1.1", trueLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); + + // isAlert == true && latitude == 48.5839 group + SearchResultGroup trueLatitudeGroup2 = trueLatitudeGroups.get(1); + Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); + List trueLatitudeGroup2Results = trueLatitudeGroup2.getResults(); + Assert.assertEquals(5, trueLatitudeGroup2Results.size()); + Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.5", trueLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.4", trueLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.3", trueLatitudeGroup2Results.get(3).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(4).getSource().get("ip_src_addr")); + } + //All Query Testcase { SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); @@ -466,6 +603,10 @@ public void test() throws Exception { } } + private int compareGroups(SearchResultGroup o1, SearchResultGroup o2) { + return o1.getKey().compareTo(o2.getKey()); + } + @AfterClass public static void stop() throws Exception { indexComponent.stop(); From a4f0ba9008f2d0ee48b8bcd639558ac955249490 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 17 Aug 2017 09:40:51 -0500 Subject: [PATCH 2/7] moved group by test cases to the end --- .../indexing/dao/SearchIntegrationTest.java | 200 +++++++++--------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index df679ff17c..5b9711d364 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -264,106 +264,6 @@ public synchronized void setup() throws Exception { @Test public void test() throws Exception { - { - SearchRequest request = JSONUtils.INSTANCE.load(sortedSizeGroupByQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - Assert.assertNull(response.getResults()); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List isAlertGroups = response.getGroups(); - Assert.assertEquals(2, isAlertGroups.size()); - Collections.sort(isAlertGroups, this::compareGroups); - - // isAlert == false group - SearchResultGroup falseGroup = isAlertGroups.get(0); - Assert.assertNull(falseGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getGroups()); - Assert.assertEquals(4, falseGroup.getTotal()); - List falseGroupResults = falseGroup.getResults(); - Assert.assertEquals(3, falseGroupResults.size()); - Assert.assertEquals(8001, falseGroupResults.get(0).getSource().get("ip_src_port")); - Assert.assertEquals(8003, falseGroupResults.get(1).getSource().get("ip_src_port")); - Assert.assertEquals(8005, falseGroupResults.get(2).getSource().get("ip_src_port")); - - // isAlert == true group - SearchResultGroup trueGroup = isAlertGroups.get(1); - Assert.assertNull(trueGroup.getGroupedBy()); - Assert.assertNull(trueGroup.getGroups()); - Assert.assertEquals(6, trueGroup.getTotal()); - List trueGroupResults = trueGroup.getResults(); - Assert.assertEquals(3, trueGroupResults.size()); - Assert.assertEquals(8002, trueGroupResults.get(0).getSource().get("ip_src_port")); - Assert.assertEquals(8004, trueGroupResults.get(1).getSource().get("ip_src_port")); - Assert.assertEquals(8006, trueGroupResults.get(2).getSource().get("ip_src_port")); - } - - { - SearchRequest request = JSONUtils.INSTANCE.load(groupByQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - Assert.assertNull(response.getResults()); - Assert.assertEquals("is_alert", response.getGroupedBy()); - List isAlertGroups = response.getGroups(); - Assert.assertEquals(2, isAlertGroups.size()); - Collections.sort(isAlertGroups, this::compareGroups); - - // isAlert == false group - SearchResultGroup falseGroup = isAlertGroups.get(0); - Assert.assertEquals("false", falseGroup.getKey()); - Assert.assertEquals("latitude", falseGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getResults()); - List falseLatitudeGroups = falseGroup.getGroups(); - Assert.assertEquals(2, falseLatitudeGroups.size()); - Collections.sort(falseLatitudeGroups, this::compareGroups); - - // isAlert == false && latitude == 48.0001 group - SearchResultGroup falseLatitudeGroup1 = falseLatitudeGroups.get(0); - Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); - List falseLatitudeGroup1Results = falseLatitudeGroup1.getResults(); - Assert.assertEquals(1, falseLatitudeGroup1Results.size()); - Assert.assertEquals("192.168.1.2", falseLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); - - // isAlert == false && latitude == 48.5839 group - SearchResultGroup falseLatitudeGroup2 = falseLatitudeGroups.get(1); - Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); - List falseLatitudeGroup2Results = falseLatitudeGroup2.getResults(); - Assert.assertEquals(3, falseLatitudeGroup2Results.size()); - Assert.assertEquals("192.168.1.8", falseLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.7", falseLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.6", falseLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); - - // isAlert == true group - SearchResultGroup trueGroup = isAlertGroups.get(1); - Assert.assertEquals("true", trueGroup.getKey()); - Assert.assertEquals("latitude", trueGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getResults()); - List trueLatitudeGroups = trueGroup.getGroups(); - Assert.assertEquals(2, trueLatitudeGroups.size()); - Collections.sort(trueLatitudeGroups, this::compareGroups); - - // isAlert == true && latitude == 48.0001 group - SearchResultGroup trueLatitudeGroup1 = trueLatitudeGroups.get(0); - Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); - List trueLatitudeGroup1Results = trueLatitudeGroup1.getResults(); - Assert.assertEquals(1, trueLatitudeGroup1Results.size()); - Assert.assertEquals("192.168.1.1", trueLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); - - // isAlert == true && latitude == 48.5839 group - SearchResultGroup trueLatitudeGroup2 = trueLatitudeGroups.get(1); - Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); - List trueLatitudeGroup2Results = trueLatitudeGroup2.getResults(); - Assert.assertEquals(5, trueLatitudeGroup2Results.size()); - Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.5", trueLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.4", trueLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.3", trueLatitudeGroup2Results.get(3).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(4).getSource().get("ip_src_addr")); - } - //All Query Testcase { SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class); @@ -601,6 +501,106 @@ public void test() throws Exception { Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } + // Group by test case + { + SearchRequest request = JSONUtils.INSTANCE.load(groupByQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + Assert.assertNull(response.getResults()); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List isAlertGroups = response.getGroups(); + Assert.assertEquals(2, isAlertGroups.size()); + Collections.sort(isAlertGroups, this::compareGroups); + + // isAlert == false group + SearchResultGroup falseGroup = isAlertGroups.get(0); + Assert.assertEquals("false", falseGroup.getKey()); + Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getResults()); + List falseLatitudeGroups = falseGroup.getGroups(); + Assert.assertEquals(2, falseLatitudeGroups.size()); + Collections.sort(falseLatitudeGroups, this::compareGroups); + + // isAlert == false && latitude == 48.0001 group + SearchResultGroup falseLatitudeGroup1 = falseLatitudeGroups.get(0); + Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); + List falseLatitudeGroup1Results = falseLatitudeGroup1.getResults(); + Assert.assertEquals(1, falseLatitudeGroup1Results.size()); + Assert.assertEquals("192.168.1.2", falseLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); + + // isAlert == false && latitude == 48.5839 group + SearchResultGroup falseLatitudeGroup2 = falseLatitudeGroups.get(1); + Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + List falseLatitudeGroup2Results = falseLatitudeGroup2.getResults(); + Assert.assertEquals(3, falseLatitudeGroup2Results.size()); + Assert.assertEquals("192.168.1.8", falseLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.7", falseLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.6", falseLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); + + // isAlert == true group + SearchResultGroup trueGroup = isAlertGroups.get(1); + Assert.assertEquals("true", trueGroup.getKey()); + Assert.assertEquals("latitude", trueGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getResults()); + List trueLatitudeGroups = trueGroup.getGroups(); + Assert.assertEquals(2, trueLatitudeGroups.size()); + Collections.sort(trueLatitudeGroups, this::compareGroups); + + // isAlert == true && latitude == 48.0001 group + SearchResultGroup trueLatitudeGroup1 = trueLatitudeGroups.get(0); + Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); + List trueLatitudeGroup1Results = trueLatitudeGroup1.getResults(); + Assert.assertEquals(1, trueLatitudeGroup1Results.size()); + Assert.assertEquals("192.168.1.1", trueLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); + + // isAlert == true && latitude == 48.5839 group + SearchResultGroup trueLatitudeGroup2 = trueLatitudeGroups.get(1); + Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); + List trueLatitudeGroup2Results = trueLatitudeGroup2.getResults(); + Assert.assertEquals(5, trueLatitudeGroup2Results.size()); + Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.5", trueLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.4", trueLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.3", trueLatitudeGroup2Results.get(3).getSource().get("ip_src_addr")); + Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(4).getSource().get("ip_src_addr")); + } + // Group by with sorting and size test case + { + SearchRequest request = JSONUtils.INSTANCE.load(sortedSizeGroupByQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + Assert.assertNull(response.getResults()); + Assert.assertEquals("is_alert", response.getGroupedBy()); + List isAlertGroups = response.getGroups(); + Assert.assertEquals(2, isAlertGroups.size()); + Collections.sort(isAlertGroups, this::compareGroups); + + // isAlert == false group + SearchResultGroup falseGroup = isAlertGroups.get(0); + Assert.assertNull(falseGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getGroups()); + Assert.assertEquals(4, falseGroup.getTotal()); + List falseGroupResults = falseGroup.getResults(); + Assert.assertEquals(3, falseGroupResults.size()); + Assert.assertEquals(8001, falseGroupResults.get(0).getSource().get("ip_src_port")); + Assert.assertEquals(8003, falseGroupResults.get(1).getSource().get("ip_src_port")); + Assert.assertEquals(8005, falseGroupResults.get(2).getSource().get("ip_src_port")); + + // isAlert == true group + SearchResultGroup trueGroup = isAlertGroups.get(1); + Assert.assertNull(trueGroup.getGroupedBy()); + Assert.assertNull(trueGroup.getGroups()); + Assert.assertEquals(6, trueGroup.getTotal()); + List trueGroupResults = trueGroup.getResults(); + Assert.assertEquals(3, trueGroupResults.size()); + Assert.assertEquals(8002, trueGroupResults.get(0).getSource().get("ip_src_port")); + Assert.assertEquals(8004, trueGroupResults.get(1).getSource().get("ip_src_port")); + Assert.assertEquals(8006, trueGroupResults.get(2).getSource().get("ip_src_port")); + } } private int compareGroups(SearchResultGroup o1, SearchResultGroup o2) { From 6d8d94f8f10c8ca2eb74f2d982de9ceaa8bc9d2d Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 24 Aug 2017 14:53:48 -0500 Subject: [PATCH 3/7] moved group function to it's own endpoint --- metron-interface/metron-rest/README.md | 31 ++- .../metron/rest/MetronRestConstants.java | 1 + .../metron/rest/config/IndexConfig.java | 10 +- .../rest/controller/SearchController.java | 13 +- .../metron/rest/service/SearchService.java | 5 +- .../rest/service/impl/SearchServiceImpl.java | 14 +- .../src/main/resources/application.yml | 1 + .../SearchControllerIntegrationTest.java | 24 +- .../elasticsearch/dao/ElasticsearchDao.java | 126 +++++---- .../ElasticsearchSearchIntegrationTest.java | 1 + .../metron/indexing/dao/AccessConfig.java | 14 +- .../apache/metron/indexing/dao/HBaseDao.java | 7 + .../apache/metron/indexing/dao/IndexDao.java | 5 +- .../metron/indexing/dao/MultiIndexDao.java | 13 + .../metron/indexing/dao/search/Group.java | 43 +++ .../indexing/dao/search/GroupOrder.java | 37 +++ .../indexing/dao/search/GroupOrderType.java | 39 +++ .../indexing/dao/search/GroupRequest.java | 48 ++++ .../indexing/dao/search/GroupResponse.java | 39 +++ ...earchResultGroup.java => GroupResult.java} | 23 +- .../indexing/dao/search/SearchRequest.java | 9 - .../indexing/dao/search/SearchResponse.java | 24 +- .../metron/indexing/dao/InMemoryDao.java | 34 ++- .../indexing/dao/SearchIntegrationTest.java | 250 +++++++++++------- 24 files changed, 562 insertions(+), 249 deletions(-) create mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java create mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java create mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java create mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java create mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java rename metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/{SearchResultGroup.java => GroupResult.java} (75%) diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index b76712bca5..b365b4d7a5 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -199,10 +199,11 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopicname)| | [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopicname)| | [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopicnamesample)| -| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)| +| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)| +| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)| | [ `GET /api/v1/search/findOne`](#get-apiv1searchfindone)| -| [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadata)| -| [ `GET /api/v1/search/search`](#get-apiv1searchcolumnmetadatacommon)| +| [ `GET /api/v1/search/column/metadata`](#get-apiv1searchcolumnmetadata)| +| [ `GET /api/v1/search/column/metadata/common`](#get-apiv1searchcolumnmetadatacommon)| | [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)| | [ `GET /api/v1/sensor/enrichment/config/list/available/enrichments`](#get-apiv1sensorenrichmentconfiglistavailableenrichments)| | [ `GET /api/v1/sensor/enrichment/config/list/available/threat/triage/aggregators`](#get-apiv1sensorenrichmentconfiglistavailablethreattriageaggregators)| @@ -353,6 +354,23 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Returns sample message * 404 - Either Kafka topic is missing or contains no messages +### `POST /api/v1/search/search` + * Description: Searches the indexing store + * Input: + * searchRequest - Search request + * Returns: + * 200 - Search response + +### `POST /api/v1/search/group` + * Description: Searches the indexing store and returns field groups. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending. + * Input: + * groupRequest - Group request + * indices - list of indices to search + * query - lucene query + * groups - List of groups (field name and sort order) + * Returns: + * 200 - Group response + ### `GET /api/v1/search/findOne` * Description: Returns latest document for a guid and sensor * Input: @@ -369,13 +387,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Document representing the output * 404 - Document with UUID and sensor type not found - -### `GET /api/v1/search/search` - * Description: Searches the indexing store - * Input: - * searchRequest - Search request - * Returns: - * 200 - Search results ### `GET /api/v1/search/column/metadata` * Description: Get column metadata for each index in the list of indicies diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index a080f77b6f..8986aa0bf2 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -61,6 +61,7 @@ public class MetronRestConstants { public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab"; public static final String SEARCH_MAX_RESULTS = "search.max.results"; + public static final String SEARCH_MAX_GROUPS = "search.max.groups"; public static final String INDEX_DAO_IMPL = "index.dao.impl"; public static final String INDEX_HBASE_TABLE_PROVIDER_IMPL = "index.hbase.provider"; } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java index 63851168a3..be031912b5 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java @@ -17,6 +17,8 @@ */ package org.apache.metron.rest.config; +import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL; + import org.apache.metron.hbase.HTableProvider; import org.apache.metron.hbase.TableProvider; import org.apache.metron.indexing.dao.AccessConfig; @@ -28,14 +30,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import org.springframework.core.env.Environment; -import java.lang.reflect.InvocationTargetException; - -import static org.apache.metron.rest.MetronRestConstants.INDEX_DAO_IMPL; -import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; - @Configuration public class IndexConfig { @@ -56,8 +52,10 @@ public IndexDao indexDao() { String hbaseProviderImpl = environment.getProperty(MetronRestConstants.INDEX_HBASE_TABLE_PROVIDER_IMPL, String.class, null); String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null); int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, -1); + int searchMaxGroups = environment.getProperty(MetronRestConstants.SEARCH_MAX_GROUPS, Integer.class, 1000); AccessConfig config = new AccessConfig(); config.setMaxSearchResults(searchMaxResults); + config.setMaxSearchGroups(searchMaxGroups); config.setGlobalConfigSupplier(() -> { try { return globalConfigService.get(); diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java index dea628c285..e21541350f 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SearchController.java @@ -21,7 +21,8 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import org.apache.metron.indexing.dao.search.GetRequest; -import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.RestException; import org.apache.metron.indexing.dao.search.SearchRequest; @@ -38,7 +39,6 @@ import java.util.Map; import java.util.Optional; import java.util.List; -import java.util.Map; @RestController @RequestMapping("/api/v1/search") @@ -54,6 +54,15 @@ ResponseEntity search(final @ApiParam(name = "searchRequest", va return new ResponseEntity<>(searchService.search(searchRequest), HttpStatus.OK); } + @ApiOperation(value = "Searches the indexing store and returns field groups. " + + "Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. " + + "The default sorting within groups is by count descending.") + @ApiResponse(message = "Group response", code = 200) + @RequestMapping(value = "/group", method = RequestMethod.POST) + ResponseEntity group(final @ApiParam(name = "groupRequest", value = "Group request", required = true) @RequestBody GroupRequest groupRequest) throws RestException { + return new ResponseEntity<>(searchService.group(groupRequest), HttpStatus.OK); + } + @ApiOperation(value = "Returns latest document for a guid and sensor") @ApiResponse(message = "Document representing the output", code = 200) @RequestMapping(value = "/findOne", method = RequestMethod.POST) diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java index ea0ae812e9..589976534d 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SearchService.java @@ -18,7 +18,8 @@ package org.apache.metron.rest.service; import org.apache.metron.indexing.dao.search.GetRequest; -import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.RestException; import org.apache.metron.indexing.dao.search.SearchRequest; @@ -27,11 +28,11 @@ import java.util.Map; import java.util.Optional; import java.util.List; -import java.util.Map; public interface SearchService { SearchResponse search(SearchRequest searchRequest) throws RestException; + GroupResponse group(GroupRequest groupRequest) throws RestException; Optional> getLatest(GetRequest request) throws RestException; Map> getColumnMetadata(List indices) throws RestException; Map getCommonColumnMetadata(List indices) throws RestException; diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index bdf6037bb6..d865e0e66d 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -19,10 +19,11 @@ import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; -import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.SearchService; @@ -34,7 +35,6 @@ import java.util.Map; import java.util.Optional; import java.util.List; -import java.util.Map; @Service public class SearchServiceImpl implements SearchService { @@ -57,6 +57,16 @@ public SearchResponse search(SearchRequest searchRequest) throws RestException { } } + @Override + public GroupResponse group(GroupRequest groupRequest) throws RestException { + try { + return dao.group(groupRequest); + } + catch(InvalidSearchException ise) { + throw new RestException(ise.getMessage(), ise); + } + } + @Override public Optional> getLatest(GetRequest request) throws RestException { try { diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml index 4aff769c14..1c50e0b8c8 100644 --- a/metron-interface/metron-rest/src/main/resources/application.yml +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -44,6 +44,7 @@ curator: search: max: results: 1000 + groups: 1000 index: dao: diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index 2e2d8f2fb7..2a6fb53857 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -172,16 +172,20 @@ public void test() throws Exception { .andExpect(jsonPath("$.responseCode").value(500)) .andExpect(jsonPath("$.message").value("Search result size must be less than 100")); - this.mockMvc.perform(post(searchUrl + "/search").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.groupByQuery)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.*", hasSize(3))) - .andExpect(jsonPath("$.total").value(10)) - .andExpect(jsonPath("$.groupedBy").value("groupByField")) - .andExpect(jsonPath("$.groups.*", hasSize(1))) - .andExpect(jsonPath("$.groups[0].key").value("groupByValue")) - .andExpect(jsonPath("$.groups[0].total").value(10)) - .andExpect(jsonPath("$.groups[0].results.*", hasSize(10))); + this.mockMvc.perform(post(searchUrl + "/group").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(SearchIntegrationTest.groupByQuery)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(2))) + .andExpect(jsonPath("$.groupedBy").value("is_alert")) + .andExpect(jsonPath("$.groupResults.*", hasSize(1))) + .andExpect(jsonPath("$.groupResults[0].*", hasSize(4))) + .andExpect(jsonPath("$.groupResults[0].key").value("is_alert_value")) + .andExpect(jsonPath("$.groupResults[0].total").value(10)) + .andExpect(jsonPath("$.groupResults[0].groupedBy").value("latitude")) + .andExpect(jsonPath("$.groupResults[0].groupResults.*", hasSize(1))) + .andExpect(jsonPath("$.groupResults[0].groupResults[0].*", hasSize(2))) + .andExpect(jsonPath("$.groupResults[0].groupResults[0].key").value("latitude_value")) + .andExpect(jsonPath("$.groupResults[0].groupResults[0].total").value(10)); this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]")) .andExpect(status().isOk()) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 08aeb39995..fd8877b217 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -36,11 +36,17 @@ import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.Group; +import org.apache.metron.indexing.dao.search.GroupOrder; +import org.apache.metron.indexing.dao.search.GroupOrderType; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; -import org.apache.metron.indexing.dao.search.SearchResultGroup; +import org.apache.metron.indexing.dao.search.GroupResult; +import org.apache.metron.indexing.dao.search.SortOrder; import org.apache.metron.indexing.dao.update.Document; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.index.IndexRequest; @@ -61,9 +67,8 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; public class ElasticsearchDao implements IndexDao { @@ -106,16 +111,11 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx } final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(new QueryStringQueryBuilder(searchRequest.getQuery())); - Optional> groupByFields = searchRequest.getGroupByFields(); - if (groupByFields.isPresent()) { - searchSourceBuilder.aggregation(getGroupsTermBuilder(searchRequest, 0)); - } else { - searchSourceBuilder.size(searchRequest.getSize()) - .from(searchRequest.getFrom()) - .fetchSource(true) - .trackScores(true); - searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); - } + searchSourceBuilder.size(searchRequest.getSize()) + .from(searchRequest.getFrom()) + .fetchSource(true) + .trackScores(true); + searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); Optional> facetFields = searchRequest.getFacetFields(); if (facetFields.isPresent()) { facetFields.get().forEach(field -> searchSourceBuilder.aggregation(new TermsBuilder(getFacentAggregationName(field)).field(field))); @@ -130,28 +130,47 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx } SearchResponse searchResponse = new SearchResponse(); searchResponse.setTotal(elasticsearchResponse.getHits().getTotalHits()); - if (!groupByFields.isPresent()) { - searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(this::getSearchResult).collect(Collectors.toList())); - } - if (groupByFields.isPresent() || facetFields.isPresent()) { + searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(this::getSearchResult).collect(Collectors.toList())); + if (facetFields.isPresent()) { Map commonColumnMetadata; try { commonColumnMetadata = getCommonColumnMetadata(searchRequest.getIndices()); } catch (IOException e) { throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(searchRequest.getIndices().toArray()))); } - if (groupByFields.isPresent()) { - searchResponse.setGroupedBy(groupByFields.get().get(0)); - searchResponse.setGroups(getGroups(groupByFields.get(), 0, elasticsearchResponse.getAggregations(), commonColumnMetadata)); - } - if (facetFields.isPresent()) { - searchResponse.setFacetCounts(getFacetCounts(facetFields.get(), elasticsearchResponse.getAggregations(), commonColumnMetadata )); - } + searchResponse.setFacetCounts(getFacetCounts(facetFields.get(), elasticsearchResponse.getAggregations(), commonColumnMetadata )); } - return searchResponse; } + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + if(client == null) { + throw new InvalidSearchException("Uninitialized Dao! You must call init() prior to use."); + } + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(new QueryStringQueryBuilder(groupRequest.getQuery())); + searchSourceBuilder.aggregation(getGroupsTermBuilder(groupRequest.getGroups(), 0)); + String[] wildcardIndices = groupRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[groupRequest.getIndices().size()]); + org.elasticsearch.action.search.SearchResponse elasticsearchResponse; + try { + elasticsearchResponse = client.search(new org.elasticsearch.action.search.SearchRequest(wildcardIndices) + .source(searchSourceBuilder)).actionGet(); + } catch (SearchPhaseExecutionException e) { + throw new InvalidSearchException("Could not execute search", e); + } + Map commonColumnMetadata; + try { + commonColumnMetadata = getCommonColumnMetadata(groupRequest.getIndices()); + } catch (IOException e) { + throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(groupRequest.getIndices().toArray()))); + } + GroupResponse groupResponse = new GroupResponse(); + groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField()); + groupResponse.setGroupResults(getGroupResults(groupRequest.getGroups(), 0, elasticsearchResponse.getAggregations(), commonColumnMetadata)); + return groupResponse; + } + @Override public synchronized void init(AccessConfig config) { if(this.client == null) { @@ -313,6 +332,14 @@ private org.elasticsearch.search.sort.SortOrder getElasticsearchSortOrder( org.elasticsearch.search.sort.SortOrder.DESC : org.elasticsearch.search.sort.SortOrder.ASC; } + private Order getElasticsearchGroupOrder(GroupOrder groupOrder) { + if (groupOrder.getGroupOrderType() == GroupOrderType.TERM) { + return groupOrder.getSortOrder() == SortOrder.ASC ? Order.term(true) : Order.term(false); + } else { + return groupOrder.getSortOrder() == SortOrder.ASC ? Order.count(true) : Order.count(false); + } + } + public Map> getFacetCounts(List fields, Aggregations aggregations, Map commonColumnMetadata) { Map> fieldCounts = new HashMap<>(); for (String field: fields) { @@ -337,43 +364,38 @@ private String formatKey(Object key, FieldType type) { } } - private TermsBuilder getGroupsTermBuilder(SearchRequest searchRequest, int index) { - List fields = searchRequest.getGroupByFields().get(); - String field = fields.get(index); - String aggregationName = getGroupByAggregationName(field); - if (index < fields.size() - 1) { - return new TermsBuilder(aggregationName).field(field) - .subAggregation(getGroupsTermBuilder(searchRequest, index + 1)); - } else { - TopHitsBuilder topHitsBuilder = new TopHitsBuilder("top_hits").setSize(searchRequest.getSize()) - .setFetchSource(true); - searchRequest.getSort().forEach(sortField -> - topHitsBuilder.addSort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); - return new TermsBuilder(aggregationName).field(field).subAggregation(topHitsBuilder); + private TermsBuilder getGroupsTermBuilder(List groups, int index) { + Group group = groups.get(index); + String aggregationName = getGroupByAggregationName(group.getField()); + TermsBuilder termsBuilder = new TermsBuilder(aggregationName) + .field(group.getField()) + .size(accessConfig.getMaxSearchGroups()) + .order(getElasticsearchGroupOrder(group.getOrder())); + if (index < groups.size() - 1) { + termsBuilder.subAggregation(getGroupsTermBuilder(groups, index + 1)); } + return termsBuilder; } - private List getGroups(List fields, int index, Aggregations aggregations, Map commonColumnMetadata) { - String field = fields.get(index); + private List getGroupResults(List groups, int index, Aggregations aggregations, Map commonColumnMetadata) { + String field = groups.get(index).getField(); Terms terms = aggregations.get(getGroupByAggregationName(field)); - List searchResultGroups = new ArrayList<>(); - if (index < fields.size() - 1) { - String childField = fields.get(index + 1); + List searchResultGroups = new ArrayList<>(); + if (index < groups.size() - 1) { + String childField = groups.get(index + 1).getField(); for(Bucket bucket: terms.getBuckets()) { - SearchResultGroup searchResultGroup = new SearchResultGroup(); - searchResultGroup.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); - searchResultGroup.setTotal(bucket.getDocCount()); - searchResultGroup.setGroupedBy(childField); - searchResultGroup.setGroups(getGroups(fields, index + 1, bucket.getAggregations(), commonColumnMetadata)); - searchResultGroups.add(searchResultGroup); + GroupResult groupResult = new GroupResult(); + groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); + groupResult.setTotal(bucket.getDocCount()); + groupResult.setGroupedBy(childField); + groupResult.setGroupResults(getGroupResults(groups, index + 1, bucket.getAggregations(), commonColumnMetadata)); + searchResultGroups.add(groupResult); } } else { for(Bucket bucket: terms.getBuckets()) { - SearchResultGroup searchResultGroup = new SearchResultGroup(); + GroupResult searchResultGroup = new GroupResult(); searchResultGroup.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); searchResultGroup.setTotal(bucket.getDocCount()); - TopHits topHits = bucket.getAggregations().get("top_hits"); - searchResultGroup.setResults(Arrays.stream(topHits.getHits().getHits()).map(this::getSearchResult).collect(Collectors.toList())); searchResultGroups.add(searchResultGroup); } } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index d794ac9d65..8090a4e6f9 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -91,6 +91,7 @@ protected IndexDao createDao() throws Exception { ret.init( new AccessConfig() {{ setMaxSearchResults(100); + setMaxSearchGroups(100); setGlobalConfigSupplier( () -> new HashMap() {{ put("es.clustername", "metron"); diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java index ddb88e5895..4f47a65228 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/AccessConfig.java @@ -17,7 +17,6 @@ */ package org.apache.metron.indexing.dao; -import org.apache.metron.hbase.HTableProvider; import org.apache.metron.hbase.TableProvider; import java.util.HashMap; @@ -26,6 +25,7 @@ public class AccessConfig { private Integer maxSearchResults; + private Integer maxSearchGroups; private Supplier> globalConfigSupplier; private Map optionalSettings = new HashMap<>(); private TableProvider tableProvider = null; @@ -54,6 +54,18 @@ public void setMaxSearchResults(Integer maxSearchResults) { this.maxSearchResults = maxSearchResults; } + /** + * The maximum search groups. + * @return + */ + public Integer getMaxSearchGroups() { + return maxSearchGroups; + } + + public void setMaxSearchGroups(Integer maxSearchGroups) { + this.maxSearchGroups = maxSearchGroups; + } + /** * Get optional settings for initializing indices. * @return diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index a1cf39884a..c890544754 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -27,6 +27,8 @@ import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; @@ -64,6 +66,11 @@ public synchronized SearchResponse search(SearchRequest searchRequest) throws In return null; } + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + return null; + } + @Override public synchronized void init(AccessConfig config) { if(this.tableInterface == null) { diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java index 350e402ed5..745dccd9a8 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java @@ -22,6 +22,8 @@ import com.flipkart.zjsonpatch.JsonPatch; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; @@ -33,7 +35,6 @@ import java.io.IOException; import org.apache.metron.indexing.dao.search.FieldType; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; @@ -49,6 +50,8 @@ public interface IndexDao { */ SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException; + GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException; + /** * Initialize the DAO with the AccessConfig object. * @param config diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java index e9a4a9a299..61c62318a1 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java @@ -22,6 +22,8 @@ import com.google.common.collect.Iterables; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; @@ -117,6 +119,17 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx return null; } + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + for(IndexDao dao : indices) { + GroupResponse s = dao.group(groupRequest); + if(s != null) { + return s; + } + } + return null; + } + @Override public void init(AccessConfig config) { for(IndexDao dao : indices) { diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java new file mode 100644 index 0000000000..be020266af --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/Group.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.metron.indexing.dao.search; + +public class Group { + + private GroupOrder order; + private String field; + + public Group() { + order = new GroupOrder(); + order.setGroupOrderType(GroupOrderType.TERM.toString()); + order.setSortOrder(SortOrder.DESC.toString()); + } + + public GroupOrder getOrder() { + return order; + } + + public void setOrder(GroupOrder order) { + this.order = order; + } + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } +} diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java new file mode 100644 index 0000000000..b90c4380e4 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrder.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.metron.indexing.dao.search; + +public class GroupOrder { + + private SortOrder sortOrder; + private GroupOrderType groupOrderType; + + public SortOrder getSortOrder() { + return sortOrder; + } + + public void setSortOrder(String sortOrder) { + this.sortOrder = SortOrder.fromString(sortOrder); + } + + public GroupOrderType getGroupOrderType() { + return groupOrderType; + } + + public void setGroupOrderType(String groupOrderType) { + this.groupOrderType = GroupOrderType.fromString(groupOrderType); + } +} diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java new file mode 100644 index 0000000000..8444e5007f --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupOrderType.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.metron.indexing.dao.search; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum GroupOrderType { + + @JsonProperty("count") + COUNT("count"), + @JsonProperty("term") + TERM("term"); + + private String groupOrderType; + + GroupOrderType(String groupOrderType) { + this.groupOrderType = groupOrderType; + } + + public String getGroupOrderType() { + return groupOrderType; + } + + public static GroupOrderType fromString(String groupOrderType) { + return GroupOrderType.valueOf(groupOrderType.toUpperCase()); + } +} diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java new file mode 100644 index 0000000000..4d09ed411c --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.metron.indexing.dao.search; + +import java.util.List; + +public class GroupRequest { + + private List indices; + private String query; + private List groups; + + public List getIndices() { + return indices; + } + + public void setIndices(List indices) { + this.indices = indices; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public List getGroups() { + return groups; + } + + public void setGroups(List groups) { + this.groups = groups; + } +} diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java new file mode 100644 index 0000000000..1b426098a1 --- /dev/null +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResponse.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.metron.indexing.dao.search; + +import java.util.List; + +public class GroupResponse { + + private String groupedBy; + private List groupResults; + + public String getGroupedBy() { + return groupedBy; + } + + public void setGroupedBy(String groupedBy) { + this.groupedBy = groupedBy; + } + + public List getGroupResults() { + return groupResults; + } + + public void setGroupResults(List groupResults) { + this.groupResults = groupResults; + } +} diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java similarity index 75% rename from metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java rename to metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java index 6a74e4da76..28e56514ef 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResultGroup.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java @@ -19,16 +19,14 @@ package org.apache.metron.indexing.dao.search; import com.fasterxml.jackson.annotation.JsonInclude; -import java.util.ArrayList; import java.util.List; -public class SearchResultGroup { +public class GroupResult { private String key; private long total; - private List results; private String groupedBy; - private List groups; + private List groupResults; public String getKey() { return key; @@ -46,15 +44,6 @@ public void setTotal(long total) { this.total = total; } - @JsonInclude(JsonInclude.Include.NON_NULL) - public List getResults() { - return results; - } - - public void setResults(List results) { - this.results = results; - } - @JsonInclude(JsonInclude.Include.NON_NULL) public String getGroupedBy() { return groupedBy; @@ -65,11 +54,11 @@ public void setGroupedBy(String groupedBy) { } @JsonInclude(JsonInclude.Include.NON_NULL) - public List getGroups() { - return groups; + public List getGroupResults() { + return groupResults; } - public void setGroups(List groups) { - this.groups = groups; + public void setGroupResults(List groups) { + this.groupResults = groups; } } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java index eadd31daa1..897f918fe3 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java @@ -28,7 +28,6 @@ public class SearchRequest { private int size; private int from; private List sort; - private List groupByFields; private List facetFields; public SearchRequest() { @@ -100,14 +99,6 @@ public void setSort(List sort) { this.sort = sort; } - public Optional> getGroupByFields() { - return groupByFields == null || groupByFields.size() == 0 ? Optional.empty() : Optional.of(groupByFields); - } - - public void setGroupByFields(List groupByFields) { - this.groupByFields = groupByFields; - } - public Optional> getFacetFields() { return facetFields == null || facetFields.size() == 0 ? Optional.empty() : Optional.of(facetFields); } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java index 88f16206bf..aad489a1e9 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchResponse.java @@ -26,9 +26,7 @@ public class SearchResponse { private long total; - private List results; - private String groupedBy; - private List groups; + private List results = new ArrayList<>(); private Map> facetCounts; /** @@ -47,7 +45,6 @@ public void setTotal(long total) { * The list of results * @return */ - @JsonInclude(JsonInclude.Include.NON_NULL) public List getResults() { return results; } @@ -56,25 +53,6 @@ public void setResults(List results) { this.results = results; } - @JsonInclude(JsonInclude.Include.NON_NULL) - public String getGroupedBy() { - return groupedBy; - } - - public void setGroupedBy(String groupedBy) { - this.groupedBy = groupedBy; - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - public List getGroups() { - return groups; - } - - public void setGroups( - List groups) { - this.groups = groups; - } - @JsonInclude(JsonInclude.Include.NON_NULL) public Map> getFacetCounts() { return facetCounts; diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java index 0dfaab56ca..6b08478bd3 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -21,9 +21,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Iterables; -import java.util.stream.Collectors; import org.apache.metron.common.Constants; -import org.apache.metron.common.configuration.writer.WriterConfiguration; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.*; import org.apache.metron.indexing.dao.update.Document; @@ -74,21 +72,29 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx finalResp.add(response.get(i)); } ret.setTotal(response.size()); - Optional> groupByFields = searchRequest.getGroupByFields(); - if (groupByFields.isPresent()) { - ret.setGroupedBy("groupByField"); - SearchResultGroup searchResultGroup = new SearchResultGroup(); - searchResultGroup.setTotal(response.size()); - searchResultGroup.setKey("groupByValue"); - searchResultGroup.setResults(finalResp); - ret.setGroups(Arrays.asList(searchResultGroup)); - } else { - ret.setResults(finalResp); - } - + ret.setResults(finalResp); return ret; } + @Override + public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException { + GroupResponse groupResponse = new GroupResponse(); + groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField()); + groupResponse.setGroupResults(getGroupResults(groupRequest.getGroups(), 0)); + return groupResponse; + } + + private List getGroupResults(List groups, int index) { + Group group = groups.get(index); + GroupResult groupResult = new GroupResult(); + groupResult.setKey(group.getField() + "_value"); + if (index < groups.size() - 1) { + groupResult.setGroupedBy(groups.get(index + 1).getField()); + groupResult.setGroupResults(getGroupResults(groups, index + 1)); + } + groupResult.setTotal(10); + return Collections.singletonList(groupResult); + } private static class ComparableComparator implements Comparator { SortOrder order = null; diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 5b9711d364..60cad217ac 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -20,11 +20,13 @@ import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.dao.search.FieldType; +import org.apache.metron.indexing.dao.search.GroupRequest; +import org.apache.metron.indexing.dao.search.GroupResponse; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; -import org.apache.metron.indexing.dao.search.SearchResultGroup; +import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.integration.InMemoryComponent; import org.junit.*; @@ -216,17 +218,16 @@ public abstract class SearchIntegrationTest { /** * { - * "groupByFields":["is_alert","latitude"], - * "indices": ["bro", "snort"], - * "query": "*", - * "from": 0, - * "size": 10, - * "sort": [ + * "groups": [ * { - * "field": "timestamp", - * "sortOrder": "desc" + * "field":"is_alert" + * }, + * { + * "field":"latitude" * } - * ] + * ], + * "indices": ["bro", "snort"], + * "query": "*" * } */ @Multiline @@ -234,21 +235,42 @@ public abstract class SearchIntegrationTest { /** * { - * "groupByFields":["is_alert"], + * "groups": [ + * { + * "field":"is_alert", + * "order": { + * "groupOrderType": "count", + * "sortOrder": "ASC" + * } + * }, + * { + * "field":"ip_src_addr", + * "order": { + * "groupOrderType": "term", + * "sortOrder": "DESC" + * } + * } + * ], * "indices": ["bro", "snort"], - * "query": "*", - * "from": 0, - * "size": 3, - * "sort": [ + * "query": "*" + * } + */ + @Multiline + public static String sortedGroupByQuery; + + /** + * { + * "groups": [ * { - * "field": "ip_src_port", - * "sortOrder": "asc" + * "field":"location_point" * } - * ] + * ], + * "indices": ["bro", "snort"], + * "query": "*" * } */ @Multiline - public static String sortedSizeGroupByQuery; + public static String badGroupQuery; protected static IndexDao dao; protected static InMemoryComponent indexComponent; @@ -501,110 +523,138 @@ public void test() throws Exception { Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } - // Group by test case + // Group by test case, default order is count descending { - SearchRequest request = JSONUtils.INSTANCE.load(groupByQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - Assert.assertNull(response.getResults()); + GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); Assert.assertEquals("is_alert", response.getGroupedBy()); - List isAlertGroups = response.getGroups(); + List isAlertGroups = response.getGroupResults(); Assert.assertEquals(2, isAlertGroups.size()); - Collections.sort(isAlertGroups, this::compareGroups); - - // isAlert == false group - SearchResultGroup falseGroup = isAlertGroups.get(0); - Assert.assertEquals("false", falseGroup.getKey()); - Assert.assertEquals("latitude", falseGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getResults()); - List falseLatitudeGroups = falseGroup.getGroups(); - Assert.assertEquals(2, falseLatitudeGroups.size()); - Collections.sort(falseLatitudeGroups, this::compareGroups); - - // isAlert == false && latitude == 48.0001 group - SearchResultGroup falseLatitudeGroup1 = falseLatitudeGroups.get(0); - Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); - Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); - List falseLatitudeGroup1Results = falseLatitudeGroup1.getResults(); - Assert.assertEquals(1, falseLatitudeGroup1Results.size()); - Assert.assertEquals("192.168.1.2", falseLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); - - // isAlert == false && latitude == 48.5839 group - SearchResultGroup falseLatitudeGroup2 = falseLatitudeGroups.get(1); - Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); - List falseLatitudeGroup2Results = falseLatitudeGroup2.getResults(); - Assert.assertEquals(3, falseLatitudeGroup2Results.size()); - Assert.assertEquals("192.168.1.8", falseLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.7", falseLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.6", falseLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); // isAlert == true group - SearchResultGroup trueGroup = isAlertGroups.get(1); + GroupResult trueGroup = isAlertGroups.get(0); Assert.assertEquals("true", trueGroup.getKey()); + Assert.assertEquals(6, trueGroup.getTotal()); Assert.assertEquals("latitude", trueGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getResults()); - List trueLatitudeGroups = trueGroup.getGroups(); + List trueLatitudeGroups = trueGroup.getGroupResults(); Assert.assertEquals(2, trueLatitudeGroups.size()); - Collections.sort(trueLatitudeGroups, this::compareGroups); + + // isAlert == true && latitude == 48.5839 group + GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); // isAlert == true && latitude == 48.0001 group - SearchResultGroup trueLatitudeGroup1 = trueLatitudeGroups.get(0); + GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); - List trueLatitudeGroup1Results = trueLatitudeGroup1.getResults(); - Assert.assertEquals(1, trueLatitudeGroup1Results.size()); - Assert.assertEquals("192.168.1.1", trueLatitudeGroup1Results.get(0).getSource().get("ip_src_addr")); - // isAlert == true && latitude == 48.5839 group - SearchResultGroup trueLatitudeGroup2 = trueLatitudeGroups.get(1); - Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); - Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); - List trueLatitudeGroup2Results = trueLatitudeGroup2.getResults(); - Assert.assertEquals(5, trueLatitudeGroup2Results.size()); - Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(0).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.5", trueLatitudeGroup2Results.get(1).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.4", trueLatitudeGroup2Results.get(2).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.3", trueLatitudeGroup2Results.get(3).getSource().get("ip_src_addr")); - Assert.assertEquals("192.168.1.1", trueLatitudeGroup2Results.get(4).getSource().get("ip_src_addr")); + // isAlert == false group + GroupResult falseGroup = isAlertGroups.get(1); + Assert.assertEquals("false", falseGroup.getKey()); + Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + List falseLatitudeGroups = falseGroup.getGroupResults(); + Assert.assertEquals(2, falseLatitudeGroups.size()); + + // isAlert == false && latitude == 48.5839 group + GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); + Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); + Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + + // isAlert == false && latitude == 48.0001 group + GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); + Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); + Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); } - // Group by with sorting and size test case + // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending { - SearchRequest request = JSONUtils.INSTANCE.load(sortedSizeGroupByQuery, SearchRequest.class); - SearchResponse response = dao.search(request); - Assert.assertEquals(10, response.getTotal()); - Assert.assertNull(response.getResults()); + GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class); + GroupResponse response = dao.group(request); Assert.assertEquals("is_alert", response.getGroupedBy()); - List isAlertGroups = response.getGroups(); + List isAlertGroups = response.getGroupResults(); Assert.assertEquals(2, isAlertGroups.size()); - Collections.sort(isAlertGroups, this::compareGroups); // isAlert == false group - SearchResultGroup falseGroup = isAlertGroups.get(0); - Assert.assertNull(falseGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getGroups()); + GroupResult falseGroup = isAlertGroups.get(0); Assert.assertEquals(4, falseGroup.getTotal()); - List falseGroupResults = falseGroup.getResults(); - Assert.assertEquals(3, falseGroupResults.size()); - Assert.assertEquals(8001, falseGroupResults.get(0).getSource().get("ip_src_port")); - Assert.assertEquals(8003, falseGroupResults.get(1).getSource().get("ip_src_port")); - Assert.assertEquals(8005, falseGroupResults.get(2).getSource().get("ip_src_port")); + Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy()); + List falseIpSrcAddrGroups = falseGroup.getGroupResults(); + Assert.assertEquals(4, falseIpSrcAddrGroups.size()); - // isAlert == true group - SearchResultGroup trueGroup = isAlertGroups.get(1); - Assert.assertNull(trueGroup.getGroupedBy()); - Assert.assertNull(trueGroup.getGroups()); + // isAlert == false && ip_src_addr == 192.168.1.8 group + GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.7 group + GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.6 group + GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.2 group + GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey()); + Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults()); + + // isAlert == false group + GroupResult trueGroup = isAlertGroups.get(1); Assert.assertEquals(6, trueGroup.getTotal()); - List trueGroupResults = trueGroup.getResults(); - Assert.assertEquals(3, trueGroupResults.size()); - Assert.assertEquals(8002, trueGroupResults.get(0).getSource().get("ip_src_port")); - Assert.assertEquals(8004, trueGroupResults.get(1).getSource().get("ip_src_port")); - Assert.assertEquals(8006, trueGroupResults.get(2).getSource().get("ip_src_port")); - } - } + Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy()); + List trueIpSrcAddrGroups = trueGroup.getGroupResults(); + Assert.assertEquals(4, trueIpSrcAddrGroups.size()); + + // isAlert == false && ip_src_addr == 192.168.1.5 group + GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0); + Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.4 group + GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1); + Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults()); - private int compareGroups(SearchResultGroup o1, SearchResultGroup o2) { - return o1.getKey().compareTo(o2.getKey()); + // isAlert == false && ip_src_addr == 192.168.1.3 group + GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2); + Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey()); + Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults()); + + // isAlert == false && ip_src_addr == 192.168.1.1 group + GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3); + Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey()); + Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy()); + Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults()); + } + //Bad group query + { + GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class); + try { + dao.group(request); + Assert.fail("Exception expected, but did not come."); + } + catch(InvalidSearchException ise) { + Assert.assertEquals("Could not execute search", ise.getMessage()); + } + } } @AfterClass From c38aa06eb7165378336925618e2d28df1b16832b Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 31 Aug 2017 11:19:26 -0500 Subject: [PATCH 4/7] added ability to calculate the sum (score) for each group --- metron-interface/metron-rest/README.md | 1 + .../SearchControllerIntegrationTest.java | 7 ++- .../elasticsearch/dao/ElasticsearchDao.java | 30 ++++++++-- .../ElasticsearchSearchIntegrationTest.java | 4 +- .../indexing/dao/search/GroupRequest.java | 10 ++++ .../indexing/dao/search/GroupResult.java | 9 +++ .../metron/indexing/dao/InMemoryDao.java | 2 + .../indexing/dao/SearchIntegrationTest.java | 56 +++++++++++-------- 8 files changed, 86 insertions(+), 33 deletions(-) diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index b365b4d7a5..82918d0610 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -367,6 +367,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available * groupRequest - Group request * indices - list of indices to search * query - lucene query + * scoreField - field used to compute a total score for each group * groups - List of groups (field name and sort order) * Returns: * 200 - Group response diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java index 2a6fb53857..645e5251ae 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SearchControllerIntegrationTest.java @@ -178,14 +178,15 @@ public void test() throws Exception { .andExpect(jsonPath("$.*", hasSize(2))) .andExpect(jsonPath("$.groupedBy").value("is_alert")) .andExpect(jsonPath("$.groupResults.*", hasSize(1))) - .andExpect(jsonPath("$.groupResults[0].*", hasSize(4))) + .andExpect(jsonPath("$.groupResults[0].*", hasSize(5))) .andExpect(jsonPath("$.groupResults[0].key").value("is_alert_value")) .andExpect(jsonPath("$.groupResults[0].total").value(10)) .andExpect(jsonPath("$.groupResults[0].groupedBy").value("latitude")) .andExpect(jsonPath("$.groupResults[0].groupResults.*", hasSize(1))) - .andExpect(jsonPath("$.groupResults[0].groupResults[0].*", hasSize(2))) + .andExpect(jsonPath("$.groupResults[0].groupResults[0].*", hasSize(3))) .andExpect(jsonPath("$.groupResults[0].groupResults[0].key").value("latitude_value")) - .andExpect(jsonPath("$.groupResults[0].groupResults[0].total").value(10)); + .andExpect(jsonPath("$.groupResults[0].groupResults[0].total").value(10)) + .andExpect(jsonPath("$.groupResults[0].groupResults[0].score").value(50)); this.mockMvc.perform(post(searchUrl + "/column/metadata").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content("[\"bro\",\"snort\"]")) .andExpect(status().isOk()) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index fd8877b217..97a33bfa6c 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -69,6 +69,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; public class ElasticsearchDao implements IndexDao { @@ -150,7 +152,7 @@ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchExcept } final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(new QueryStringQueryBuilder(groupRequest.getQuery())); - searchSourceBuilder.aggregation(getGroupsTermBuilder(groupRequest.getGroups(), 0)); + searchSourceBuilder.aggregation(getGroupsTermBuilder(groupRequest, 0)); String[] wildcardIndices = groupRequest.getIndices().stream().map(index -> String.format("%s*", index)).toArray(value -> new String[groupRequest.getIndices().size()]); org.elasticsearch.action.search.SearchResponse elasticsearchResponse; try { @@ -167,7 +169,7 @@ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchExcept } GroupResponse groupResponse = new GroupResponse(); groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField()); - groupResponse.setGroupResults(getGroupResults(groupRequest.getGroups(), 0, elasticsearchResponse.getAggregations(), commonColumnMetadata)); + groupResponse.setGroupResults(getGroupResults(groupRequest, 0, elasticsearchResponse.getAggregations(), commonColumnMetadata)); return groupResponse; } @@ -364,7 +366,8 @@ private String formatKey(Object key, FieldType type) { } } - private TermsBuilder getGroupsTermBuilder(List groups, int index) { + private TermsBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) { + List groups = groupRequest.getGroups(); Group group = groups.get(index); String aggregationName = getGroupByAggregationName(group.getField()); TermsBuilder termsBuilder = new TermsBuilder(aggregationName) @@ -372,12 +375,18 @@ private TermsBuilder getGroupsTermBuilder(List groups, int index) { .size(accessConfig.getMaxSearchGroups()) .order(getElasticsearchGroupOrder(group.getOrder())); if (index < groups.size() - 1) { - termsBuilder.subAggregation(getGroupsTermBuilder(groups, index + 1)); + termsBuilder.subAggregation(getGroupsTermBuilder(groupRequest, index + 1)); + } else { + Optional scoreField = groupRequest.getScoreField(); + if (scoreField.isPresent()) { + termsBuilder.subAggregation(new SumBuilder(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0)); + } } return termsBuilder; } - private List getGroupResults(List groups, int index, Aggregations aggregations, Map commonColumnMetadata) { + private List getGroupResults(GroupRequest groupRequest, int index, Aggregations aggregations, Map commonColumnMetadata) { + List groups = groupRequest.getGroups(); String field = groups.get(index).getField(); Terms terms = aggregations.get(getGroupByAggregationName(field)); List searchResultGroups = new ArrayList<>(); @@ -388,7 +397,7 @@ private List getGroupResults(List groups, int index, Aggrega groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); groupResult.setTotal(bucket.getDocCount()); groupResult.setGroupedBy(childField); - groupResult.setGroupResults(getGroupResults(groups, index + 1, bucket.getAggregations(), commonColumnMetadata)); + groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata)); searchResultGroups.add(groupResult); } } else { @@ -396,6 +405,11 @@ private List getGroupResults(List groups, int index, Aggrega GroupResult searchResultGroup = new GroupResult(); searchResultGroup.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); searchResultGroup.setTotal(bucket.getDocCount()); + Optional scoreField = groupRequest.getScoreField(); + if (scoreField.isPresent()) { + Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get())); + searchResultGroup.setScore(score.getValue()); + } searchResultGroups.add(searchResultGroup); } } @@ -418,4 +432,8 @@ private String getFacentAggregationName(String field) { private String getGroupByAggregationName(String field) { return String.format("%s_group", field); } + + private String getSumAggregationName(String field) { + return String.format("%s_score", field); + } } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 8090a4e6f9..5de9fd2a1f 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -50,7 +50,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "long_field": { "type": "long" }, * "timestamp" : { "type": "date" }, * "latitude" : { "type": "float" }, - * "double_field": { "type": "double" }, + * "score": { "type": "double" }, * "is_alert": { "type": "boolean" }, * "location_point": { "type": "geo_point" }, * "bro_field": { "type": "string" }, @@ -72,7 +72,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { * "long_field": { "type": "long" }, * "timestamp" : { "type": "date" }, * "latitude" : { "type": "float" }, - * "double_field": { "type": "double" }, + * "score": { "type": "double" }, * "is_alert": { "type": "boolean" }, * "location_point": { "type": "geo_point" }, * "snort_field": { "type": "integer" }, diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java index 4d09ed411c..121da1004c 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupRequest.java @@ -15,11 +15,13 @@ package org.apache.metron.indexing.dao.search; import java.util.List; +import java.util.Optional; public class GroupRequest { private List indices; private String query; + private String scoreField; private List groups; public List getIndices() { @@ -38,6 +40,14 @@ public void setQuery(String query) { this.query = query; } + public Optional getScoreField() { + return scoreField == null ? Optional.empty() : Optional.of(scoreField); + } + + public void setScoreField(String scoreField) { + this.scoreField = scoreField; + } + public List getGroups() { return groups; } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java index 28e56514ef..d40f1462e4 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/GroupResult.java @@ -25,6 +25,7 @@ public class GroupResult { private String key; private long total; + private Double score; private String groupedBy; private List groupResults; @@ -44,6 +45,14 @@ public void setTotal(long total) { this.total = total; } + public Double getScore() { + return score; + } + + public void setScore(Double score) { + this.score = score; + } + @JsonInclude(JsonInclude.Include.NON_NULL) public String getGroupedBy() { return groupedBy; diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java index 6b08478bd3..6e48b584e1 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java @@ -91,6 +91,8 @@ private List getGroupResults(List groups, int index) { if (index < groups.size() - 1) { groupResult.setGroupedBy(groups.get(index + 1).getField()); groupResult.setGroupResults(getGroupResults(groups, index + 1)); + } else { + groupResult.setScore(50.0); } groupResult.setTotal(10); return Collections.singletonList(groupResult); diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 60cad217ac..6ccafb1981 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -39,11 +39,11 @@ public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "double_field": 1.00001, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "double_field": 1.00002, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "double_field": 1.00002, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "double_field": 1.00002, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4"}, - * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "double_field": 1.00001, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5"} + * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.2", "ip_src_port": 8009, "long_field": 20000, "timestamp":2, "latitude": 48.0001, "score": 50.0, "is_alert":false, "location_point": "48.5839,7.7455", "bro_field": "bro data 2", "duplicate_name_field": "data 2"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.3", "ip_src_port": 8008, "long_field": 10000, "timestamp":3, "latitude": 48.5839, "score": 20.0, "is_alert":true, "location_point": "50.0,7.7455", "bro_field": "bro data 3", "duplicate_name_field": "data 3"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.4", "ip_src_port": 8007, "long_field": 10000, "timestamp":4, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 4", "duplicate_name_field": "data 4"}, + * {"source:type": "bro", "ip_src_addr":"192.168.1.5", "ip_src_port": 8006, "long_field": 10000, "timestamp":5, "latitude": 48.5839, "score": 98.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 5", "duplicate_name_field": "data 5"} * ] */ @Multiline @@ -51,11 +51,11 @@ public abstract class SearchIntegrationTest { /** * [ - * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "double_field": 1.00001, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "double_field": 1.00002, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "double_field": 1.00001, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "double_field": 1.00002, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4}, - * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "double_field": 1.00001, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5} + * {"source:type": "snort", "ip_src_addr":"192.168.1.6", "ip_src_port": 8005, "long_field": 10000, "timestamp":6, "latitude": 48.5839, "score": 50.0, "is_alert":false, "location_point": "50.0,7.7455", "snort_field": 10, "duplicate_name_field": 1}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8004, "long_field": 10000, "timestamp":7, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 20, "duplicate_name_field": 2}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.7", "ip_src_port": 8003, "long_field": 10000, "timestamp":8, "latitude": 48.5839, "score": 20.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 30, "duplicate_name_field": 3}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.1", "ip_src_port": 8002, "long_field": 20000, "timestamp":9, "latitude": 48.0001, "score": 50.0, "is_alert":true, "location_point": "48.5839,7.7455", "snort_field": 40, "duplicate_name_field": 4}, + * {"source:type": "snort", "ip_src_addr":"192.168.1.8", "ip_src_port": 8001, "long_field": 10000, "timestamp":10, "latitude": 48.5839, "score": 10.0, "is_alert":false, "location_point": "48.5839,7.7455", "snort_field": 50, "duplicate_name_field": 5} * ] */ @Multiline @@ -148,7 +148,7 @@ public abstract class SearchIntegrationTest { /** * { - * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "double_field", "is_alert"], + * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"], * "indices": ["bro", "snort"], * "query": "*", * "from": 0, @@ -226,6 +226,7 @@ public abstract class SearchIntegrationTest { * "field":"latitude" * } * ], + * "scoreField":"score", * "indices": ["bro", "snort"], * "query": "*" * } @@ -406,14 +407,18 @@ public void test() throws Exception { Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001); Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0))); Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1))); - Map doubleFieldCounts = facetCounts.get("double_field"); - Assert.assertEquals(2, doubleFieldCounts.size()); - List doubleFieldKeys = new ArrayList<>(doubleFieldCounts.keySet()); - Collections.sort(doubleFieldKeys); - Assert.assertEquals(1.00001, Double.parseDouble(doubleFieldKeys.get(0)), 0.00001); - Assert.assertEquals(1.00002, Double.parseDouble(doubleFieldKeys.get(1)), 0.00001); - Assert.assertEquals(new Long(5), doubleFieldCounts.get(doubleFieldKeys.get(0))); - Assert.assertEquals(new Long(5), doubleFieldCounts.get(doubleFieldKeys.get(1))); + Map scoreFieldCounts = facetCounts.get("score"); + Assert.assertEquals(4, scoreFieldCounts.size()); + List scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet()); + Collections.sort(scoreFieldKeys); + Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001); + Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001); + Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001); + Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001); + Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0))); + Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1))); + Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2))); + Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3))); Map isAlertCounts = facetCounts.get("is_alert"); Assert.assertEquals(2, isAlertCounts.size()); Assert.assertEquals(new Long(6), isAlertCounts.get("true")); @@ -459,7 +464,7 @@ public void test() throws Exception { Assert.assertEquals(FieldType.LONG, broTypes.get("long_field")); Assert.assertEquals(FieldType.DATE, broTypes.get("timestamp")); Assert.assertEquals(FieldType.FLOAT, broTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, broTypes.get("double_field")); + Assert.assertEquals(FieldType.DOUBLE, broTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, broTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, broTypes.get("location_point")); Assert.assertEquals(FieldType.STRING, broTypes.get("bro_field")); @@ -472,7 +477,7 @@ public void test() throws Exception { Assert.assertEquals(FieldType.LONG, snortTypes.get("long_field")); Assert.assertEquals(FieldType.DATE, snortTypes.get("timestamp")); Assert.assertEquals(FieldType.FLOAT, snortTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("double_field")); + Assert.assertEquals(FieldType.DOUBLE, snortTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, snortTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, snortTypes.get("location_point")); Assert.assertEquals(FieldType.INTEGER, snortTypes.get("snort_field")); @@ -505,7 +510,7 @@ public void test() throws Exception { Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field")); Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp")); Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude")); - Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("double_field")); + Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score")); Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert")); Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point")); } @@ -536,23 +541,28 @@ public void test() throws Exception { Assert.assertEquals("true", trueGroup.getKey()); Assert.assertEquals(6, trueGroup.getTotal()); Assert.assertEquals("latitude", trueGroup.getGroupedBy()); + Assert.assertNull(trueGroup.getScore()); List trueLatitudeGroups = trueGroup.getGroupResults(); Assert.assertEquals(2, trueLatitudeGroups.size()); + // isAlert == true && latitude == 48.5839 group GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0); Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001); Assert.assertEquals(5, trueLatitudeGroup2.getTotal()); + Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001); // isAlert == true && latitude == 48.0001 group GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1); Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001); Assert.assertEquals(1, trueLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001); // isAlert == false group GroupResult falseGroup = isAlertGroups.get(1); Assert.assertEquals("false", falseGroup.getKey()); Assert.assertEquals("latitude", falseGroup.getGroupedBy()); + Assert.assertNull(falseGroup.getScore()); List falseLatitudeGroups = falseGroup.getGroupResults(); Assert.assertEquals(2, falseLatitudeGroups.size()); @@ -560,11 +570,13 @@ public void test() throws Exception { GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0); Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001); Assert.assertEquals(3, falseLatitudeGroup2.getTotal()); + Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001); // isAlert == false && latitude == 48.0001 group GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1); Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001); Assert.assertEquals(1, falseLatitudeGroup1.getTotal()); + Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001); } // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending { From 6cc7ae14ad31525918ebf5565ad4fbac6e7a0592 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 31 Aug 2017 11:51:39 -0500 Subject: [PATCH 5/7] resolved merge conflicts --- .../elasticsearch/dao/ElasticsearchDao.java | 76 ++++--------------- 1 file changed, 15 insertions(+), 61 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 7c3c6aeff7..7ba2bda408 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -41,11 +41,11 @@ import org.apache.metron.indexing.dao.search.GroupOrderType; import org.apache.metron.indexing.dao.search.GroupRequest; import org.apache.metron.indexing.dao.search.GroupResponse; +import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.indexing.dao.search.InvalidSearchException; import org.apache.metron.indexing.dao.search.SearchRequest; import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; -import org.apache.metron.indexing.dao.search.GroupResult; import org.apache.metron.indexing.dao.search.SortOrder; import org.apache.metron.indexing.dao.update.Document; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; @@ -72,29 +72,6 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.*; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; public class ElasticsearchDao implements IndexDao { private transient TransportClient client; @@ -134,34 +111,18 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx if (searchRequest.getSize() > accessConfig.getMaxSearchResults()) { throw new InvalidSearchException("Search result size must be less than " + accessConfig.getMaxSearchResults()); } - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(new QueryStringQueryBuilder(searchRequest.getQuery())); - searchSourceBuilder.size(searchRequest.getSize()) - .from(searchRequest.getFrom()) - .fetchSource(true) - .trackScores(true); - searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .size(searchRequest.getSize()) .from(searchRequest.getFrom()) .query(new QueryStringQueryBuilder(searchRequest.getQuery())) - .trackScores(true); + searchRequest.getSort().forEach(sortField -> searchSourceBuilder.sort(sortField.getField(), getElasticsearchSortOrder(sortField.getSortOrder()))); Optional> fields = searchRequest.getFields(); if (fields.isPresent()) { searchSourceBuilder.fields(fields.get()); } else { searchSourceBuilder.fetchSource(true); } - for (SortField sortField : searchRequest.getSort()) { - FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(sortField.getField()); - if (sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC) { - fieldSortBuilder.order(org.elasticsearch.search.sort.SortOrder.DESC); - } else { - fieldSortBuilder.order(org.elasticsearch.search.sort.SortOrder.ASC); - } - searchSourceBuilder = searchSourceBuilder.sort(fieldSortBuilder); - } Optional> facetFields = searchRequest.getFacetFields(); if (facetFields.isPresent()) { facetFields.get().forEach(field -> searchSourceBuilder.aggregation(new TermsBuilder(getFacentAggregationName(field)).field(field))); @@ -176,24 +137,8 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx } SearchResponse searchResponse = new SearchResponse(); searchResponse.setTotal(elasticsearchResponse.getHits().getTotalHits()); - searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(this::getSearchResult).collect(Collectors.toList())); - searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> { - SearchResult searchResult = new SearchResult(); - searchResult.setId(searchHit.getId()); - Map source; - if (fields.isPresent()) { - source = new HashMap<>(); - searchHit.getFields().forEach((key, value) -> { - source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues()); - }); - } else { - source = searchHit.getSource(); - } - searchResult.setSource(source); - searchResult.setScore(searchHit.getScore()); - searchResult.setIndex(searchHit.getIndex()); - return searchResult; - }).collect(Collectors.toList())); + searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> + getSearchResult(searchHit, fields.isPresent())).collect(Collectors.toList())); if (facetFields.isPresent()) { Map commonColumnMetadata; try { @@ -477,10 +422,19 @@ private List getGroupResults(GroupRequest groupRequest, int index, return searchResultGroups; } - private SearchResult getSearchResult(SearchHit searchHit) { + private SearchResult getSearchResult(SearchHit searchHit, boolean fieldsPresent) { SearchResult searchResult = new SearchResult(); searchResult.setId(searchHit.getId()); - searchResult.setSource(searchHit.getSource()); + Map source; + if (fieldsPresent) { + source = new HashMap<>(); + searchHit.getFields().forEach((key, value) -> { + source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues()); + }); + } else { + source = searchHit.getSource(); + } + searchResult.setSource(source); searchResult.setScore(searchHit.getScore()); searchResult.setIndex(searchHit.getIndex()); return searchResult; From 13f07ec27f142a930c9e942c9bb6cf85cae40547 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Fri, 1 Sep 2017 13:34:40 -0500 Subject: [PATCH 6/7] added score to all group levels --- .../elasticsearch/dao/ElasticsearchDao.java | 41 ++++++++----------- .../indexing/dao/SearchIntegrationTest.java | 4 +- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 7ba2bda408..c71da0d5c0 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -382,11 +382,10 @@ private TermsBuilder getGroupsTermBuilder(GroupRequest groupRequest, int index) .order(getElasticsearchGroupOrder(group.getOrder())); if (index < groups.size() - 1) { termsBuilder.subAggregation(getGroupsTermBuilder(groupRequest, index + 1)); - } else { - Optional scoreField = groupRequest.getScoreField(); - if (scoreField.isPresent()) { - termsBuilder.subAggregation(new SumBuilder(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0)); - } + } + Optional scoreField = groupRequest.getScoreField(); + if (scoreField.isPresent()) { + termsBuilder.subAggregation(new SumBuilder(getSumAggregationName(scoreField.get())).field(scoreField.get()).missing(0)); } return termsBuilder; } @@ -396,28 +395,20 @@ private List getGroupResults(GroupRequest groupRequest, int index, String field = groups.get(index).getField(); Terms terms = aggregations.get(getGroupByAggregationName(field)); List searchResultGroups = new ArrayList<>(); - if (index < groups.size() - 1) { - String childField = groups.get(index + 1).getField(); - for(Bucket bucket: terms.getBuckets()) { - GroupResult groupResult = new GroupResult(); - groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); - groupResult.setTotal(bucket.getDocCount()); - groupResult.setGroupedBy(childField); - groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata)); - searchResultGroups.add(groupResult); + for(Bucket bucket: terms.getBuckets()) { + GroupResult groupResult = new GroupResult(); + groupResult.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); + groupResult.setTotal(bucket.getDocCount()); + Optional scoreField = groupRequest.getScoreField(); + if (scoreField.isPresent()) { + Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get())); + groupResult.setScore(score.getValue()); } - } else { - for(Bucket bucket: terms.getBuckets()) { - GroupResult searchResultGroup = new GroupResult(); - searchResultGroup.setKey(formatKey(bucket.getKey(), commonColumnMetadata.get(field))); - searchResultGroup.setTotal(bucket.getDocCount()); - Optional scoreField = groupRequest.getScoreField(); - if (scoreField.isPresent()) { - Sum score = bucket.getAggregations().get(getSumAggregationName(scoreField.get())); - searchResultGroup.setScore(score.getValue()); - } - searchResultGroups.add(searchResultGroup); + if (index < groups.size() - 1) { + groupResult.setGroupedBy(groups.get(index + 1).getField()); + groupResult.setGroupResults(getGroupResults(groupRequest, index + 1, bucket.getAggregations(), commonColumnMetadata)); } + searchResultGroups.add(groupResult); } return searchResultGroups; } diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 112481563d..0db8e37623 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -604,7 +604,7 @@ public void test() throws Exception { Assert.assertEquals("true", trueGroup.getKey()); Assert.assertEquals(6, trueGroup.getTotal()); Assert.assertEquals("latitude", trueGroup.getGroupedBy()); - Assert.assertNull(trueGroup.getScore()); + Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001); List trueLatitudeGroups = trueGroup.getGroupResults(); Assert.assertEquals(2, trueLatitudeGroups.size()); @@ -625,7 +625,7 @@ public void test() throws Exception { GroupResult falseGroup = isAlertGroups.get(1); Assert.assertEquals("false", falseGroup.getKey()); Assert.assertEquals("latitude", falseGroup.getGroupedBy()); - Assert.assertNull(falseGroup.getScore()); + Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001); List falseLatitudeGroups = falseGroup.getGroupResults(); Assert.assertEquals(2, falseLatitudeGroups.size()); From fff03c2de8c98899b434999fc26d30d287f4fb69 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Fri, 8 Sep 2017 16:25:33 -0500 Subject: [PATCH 7/7] addressing PR feedback --- metron-interface/metron-rest/README.md | 2 +- .../main/java/org/apache/metron/rest/config/IndexConfig.java | 2 +- .../org/apache/metron/elasticsearch/dao/ElasticsearchDao.java | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 82918d0610..d32e9fd37b 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -362,7 +362,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - Search response ### `POST /api/v1/search/group` - * Description: Searches the indexing store and returns field groups. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending. + * Description: Searches the indexing store and returns field groups. Groups are hierarchical and nested in the order the fields appear in the 'groups' request parameter. The default sorting within groups is by count descending. A groupOrder type of count will sort based on then number of documents in a group while a groupType of term will sort by the groupBy term. * Input: * groupRequest - Group request * indices - list of indices to search diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java index be031912b5..b6ac5e775e 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/IndexConfig.java @@ -51,7 +51,7 @@ public IndexDao indexDao() { try { String hbaseProviderImpl = environment.getProperty(MetronRestConstants.INDEX_HBASE_TABLE_PROVIDER_IMPL, String.class, null); String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null); - int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, -1); + int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, 1000); int searchMaxGroups = environment.getProperty(MetronRestConstants.SEARCH_MAX_GROUPS, Integer.class, 1000); AccessConfig config = new AccessConfig(); config.setMaxSearchResults(searchMaxResults); diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index c71da0d5c0..0d7a76c36b 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -156,6 +156,9 @@ public GroupResponse group(GroupRequest groupRequest) throws InvalidSearchExcept if(client == null) { throw new InvalidSearchException("Uninitialized Dao! You must call init() prior to use."); } + if (groupRequest.getGroups() == null || groupRequest.getGroups().size() == 0) { + throw new InvalidSearchException("At least 1 group must be provided."); + } final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(new QueryStringQueryBuilder(groupRequest.getQuery())); searchSourceBuilder.aggregation(getGroupsTermBuilder(groupRequest, 0));