From 89d1d5742005d7869728ae7a67c96acc26ce651b Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Mon, 12 Nov 2018 12:24:40 -0500 Subject: [PATCH 1/2] METRON-1867 Remove `/api/v1/update/replace` endpoint --- .../src/app/model/replace-request.ts | 24 -- metron-interface/metron-rest/README.md | 26 -- .../rest/controller/UpdateController.java | 12 - .../metron/rest/service/UpdateService.java | 2 - .../rest/service/impl/UpdateServiceImpl.java | 10 - .../UpdateControllerIntegrationTest.java | 116 +++--- .../elasticsearch/dao/ElasticsearchDao.java | 17 +- .../apache/metron/indexing/dao/HBaseDao.java | 23 +- .../metron/indexing/dao/MultiIndexDao.java | 17 +- .../AbstractLuceneMetaAlertUpdateDao.java | 28 +- .../indexing/dao/update/ReplaceRequest.java | 77 ---- .../metron/indexing/dao/update/UpdateDao.java | 22 +- .../indexing/dao/InMemoryMetaAlertDao.java | 23 +- .../indexing/dao/UpdateIntegrationTest.java | 369 ++++++++---------- .../integration/HBaseDaoIntegrationTest.java | 32 +- .../org/apache/metron/solr/dao/SolrDao.java | 15 +- .../metron/solr/dao/SolrMetaAlertDao.java | 14 +- .../apache/metron/solr/dao/SolrUpdateDao.java | 23 +- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 17 +- .../SolrUpdateIntegrationTest.java | 17 +- 20 files changed, 322 insertions(+), 562 deletions(-) delete mode 100644 metron-interface/metron-alerts/src/app/model/replace-request.ts delete mode 100644 metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/ReplaceRequest.java diff --git a/metron-interface/metron-alerts/src/app/model/replace-request.ts b/metron-interface/metron-alerts/src/app/model/replace-request.ts deleted file mode 100644 index d7c18b22f6..0000000000 --- a/metron-interface/metron-alerts/src/app/model/replace-request.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export class ReplaceRequest { - replacement: {[key: string]: any}; - guid: string; - sensorType: string; - index: string; -} diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index f85707f3c9..a3cbad75d7 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -373,7 +373,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/storm/{name}`](#get-apiv1stormname)| | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)| | [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)| -| [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)| | [ `POST /api/v1/update/add/comment`](#put-apiv1updateaddcomment)| | [ `POST /api/v1/update/remove/comment`](#put-apiv1updateremovecomment)| | [ `GET /api/v1/user`](#get-apiv1user)| @@ -978,31 +977,6 @@ Request and Response objects are JSON formatted. The JSON schemas are available * Returns: * 200 - Nothing * 404 - Document not found - -### `PUT /api/v1/update/replace` - * Description: Replace a document - * Input: - * request - Replacement request - * guid - The Patch UUID - * sensorType - The sensor type - * replacement - A Map representing the replaced document - * Example replacing a `bro` message with guid of `000-000-0000` : - ``` - { - "guid" : "000-000-0000", - "sensorType" : "bro", - "replacement" : { - "source:type": "bro", - "guid" : "bro_index_2017.01.01.01:1", - "ip_src_addr":"192.168.1.2", - "ip_src_port": 8009, - "timestamp":200, - "rejected":false - } - } - ``` - * Returns: - * 200 - Current user ### `POST /api/v1/update/add/comment` * Description: Add a comment to an alert diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java index 5550358242..f16b6cdb8e 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java @@ -24,7 +24,6 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.UpdateService; import org.springframework.beans.factory.annotation.Autowired; @@ -57,17 +56,6 @@ ResponseEntity patch( } } - @ApiOperation(value = "Replace a document with a full replacement") - @ApiResponse(message = "Returns the complete replaced document.", code = 200) - @RequestMapping(value = "/replace", method = RequestMethod.POST) - ResponseEntity replace( - final @ApiParam(name = "request", value = "Replacement request", required = true) - @RequestBody - ReplaceRequest request - ) throws RestException { - return new ResponseEntity<>(service.replace(request), HttpStatus.OK); - } - @ApiOperation(value = "Add a comment to an alert") @ApiResponse(message = "Returns the complete alert document with comments added.", code = 200) @RequestMapping(value = "/add/comment", method = RequestMethod.POST) diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java index 19b3485175..c3729c37fa 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java @@ -21,13 +21,11 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.apache.metron.rest.RestException; public interface UpdateService { Document patch(PatchRequest request) throws RestException, OriginalNotFoundException; - Document replace(ReplaceRequest request) throws RestException; Document addComment(CommentAddRemoveRequest request) throws RestException; Document removeComment(CommentAddRemoveRequest request) throws RestException; } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java index 63dd4c0036..8b159da909 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java @@ -22,7 +22,6 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.UpdateService; import org.slf4j.Logger; @@ -52,15 +51,6 @@ public Document patch(PatchRequest request) throws RestException, OriginalNotFou } } - @Override - public Document replace(ReplaceRequest request) throws RestException { - try { - return dao.replace(request, Optional.of(System.currentTimeMillis())); - } catch (Exception e) { - throw new RestException(e.getMessage(), e); - } - } - @Override public Document addComment(CommentAddRemoveRequest request) throws RestException { try { diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java index 6b8d5d36a9..7995557695 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java @@ -17,28 +17,15 @@ */ package org.apache.metron.rest.controller; -import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; -import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - import com.google.common.collect.ImmutableMap; -import java.util.NavigableMap; import org.adrianwalker.multilinestring.Multiline; import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; -import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.HBaseDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; -import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.rest.service.UpdateService; import org.junit.Assert; @@ -51,11 +38,23 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.ResultActions; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.springframework.web.context.WebApplicationContext; +import java.util.NavigableMap; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles(TEST_PROFILE) @@ -101,23 +100,6 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest { @Multiline public static String patch; - /** - { - "guid" : "bro_2", - "sensorType" : "bro", - "replacement" : { - "source:type": "bro", - "guid" : "bro_2", - "ip_src_addr":"192.168.1.2", - "ip_src_port": 8009, - "timestamp":200, - "rejected":false - } - } - */ - @Multiline - public static String replace; - /** { "guid" : "bro_2", @@ -154,38 +136,50 @@ public void setup() throws Exception { } @Test - public void test() throws Exception { + public void shouldPatchDocument() throws Exception { String guid = "bro_2"; - ResultActions result = this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)); - try { - result.andExpect(status().isOk()) + + // request used to find the message + MockHttpServletRequestBuilder findOneRequest = post(searchUrl + "/findOne") + .with(httpBasic(user, password)) + .with(csrf()) + .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) + .content(findMessage0); + + // request used to patch the document + MockHttpServletRequestBuilder patchRequest = patch(updateUrl + "/patch") + .with(httpBasic(user, password)) + .with(csrf()) + .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) + .content(patch); + + // the document should exist, but without the 'project' field defined + this.mockMvc.perform(findOneRequest) + .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.source:type").value("bro")) .andExpect(jsonPath("$.guid").value(guid)) .andExpect(jsonPath("$.project").doesNotExist()) - .andExpect(jsonPath("$.timestamp").value(2)) - ; - } - catch(Throwable t) { - System.err.println(result.andReturn().getResponse().getContentAsString()); - throw t; - } + .andExpect(jsonPath("$.timestamp").value(2)); + + // nothing is recorded in HBase MockHTable table = (MockHTable) MockHBaseTableProvider.getFromCache(TABLE); Assert.assertEquals(0,table.size()); - this.mockMvc.perform(patch(updateUrl+ "/patch").with(httpBasic(user, password)) - .with(csrf()) - .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) - .content(patch) - ) + + // patch the document + this.mockMvc.perform(patchRequest) .andExpect(status().isOk()); - this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)) + + // the document should now have the 'project' field + this.mockMvc.perform(findOneRequest) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) .andExpect(jsonPath("$.source:type").value("bro")) .andExpect(jsonPath("$.guid").value(guid)) .andExpect(jsonPath("$.project").value("metron")) - .andExpect(jsonPath("$.timestamp").value(2)) - ; + .andExpect(jsonPath("$.timestamp").value(2)); + + // the change should be recorded in HBase Assert.assertEquals(1,table.size()); { //ensure hbase is up to date @@ -194,28 +188,6 @@ public void test() throws Exception { NavigableMap columns = r.getFamilyMap(CF.getBytes()); Assert.assertEquals(1, columns.size()); } - this.mockMvc.perform(post(updateUrl+ "/replace").with(httpBasic(user, password)) - .with(csrf()) - .contentType(MediaType.parseMediaType("application/json;charset=UTF-8")) - .content(replace) - ) - .andExpect(status().isOk()); - this.mockMvc.perform(post(searchUrl + "/findOne").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(findMessage0)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) - .andExpect(jsonPath("$.source:type").value("bro")) - .andExpect(jsonPath("$.guid").value(guid)) - .andExpect(jsonPath("$.project").doesNotExist()) - .andExpect(jsonPath("$.timestamp").value(200)) - ; - Assert.assertEquals(1,table.size()); - { - //ensure hbase is up to date - Get g = new Get(new HBaseDao.Key(guid, "bro").toBytes()); - Result r = table.get(g); - NavigableMap columns = r.getFamilyMap(CF.getBytes()); - Assert.assertEquals(2, columns.size()); - } } @Test diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index dcd6fdb54b..675d22f190 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -17,11 +17,6 @@ */ package org.apache.metron.elasticsearch.dao; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; -import java.util.Map; -import java.util.Optional; import org.apache.metron.elasticsearch.client.ElasticsearchClient; import org.apache.metron.elasticsearch.client.ElasticsearchClientFactory; import org.apache.metron.indexing.dao.AccessConfig; @@ -38,12 +33,17 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.index.query.QueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Map; +import java.util.Optional; + public class ElasticsearchDao implements IndexDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -155,11 +155,6 @@ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, return updateDao.patch(retrieveLatestDao, request, timestamp); } - @Override - public Document replace(ReplaceRequest request, Optional timestamp) throws IOException { - return updateDao.replace(request, timestamp); - } - @Override public Document addCommentToAlert(CommentAddRemoveRequest request) throws IOException { return updateDao.addCommentToAlert(request); diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index 71d05444c9..9bdb3ceec1 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -18,17 +18,6 @@ package org.apache.metron.indexing.dao; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.stream.Collectors; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; @@ -48,6 +37,18 @@ import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.stream.Collectors; + /** * The HBaseDao is an index dao which only supports the following actions: * * Update diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java index c3e2108e99..f41a82f0ae 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java @@ -20,14 +20,6 @@ import com.google.common.base.Joiner; import com.google.common.collect.Iterables; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; @@ -39,6 +31,15 @@ import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + public class MultiIndexDao implements IndexDao { private List indices; diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java index 096baf174b..39396f05ad 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java @@ -18,7 +18,19 @@ package org.apache.metron.indexing.dao.metaalert.lucene; -import static org.apache.metron.common.Constants.GUID; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; +import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao; +import org.apache.metron.indexing.dao.metaalert.MetaScores; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.indexing.dao.update.UpdateDao; import java.io.IOException; import java.util.ArrayList; @@ -34,19 +46,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import org.apache.metron.common.Constants; -import org.apache.metron.indexing.dao.RetrieveLatestDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; -import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; -import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; -import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao; -import org.apache.metron.indexing.dao.metaalert.MetaScores; -import org.apache.metron.indexing.dao.search.GetRequest; -import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.OriginalNotFoundException; -import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.UpdateDao; +import static org.apache.metron.common.Constants.GUID; public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdateDao { diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/ReplaceRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/ReplaceRequest.java deleted file mode 100644 index 96bca3028e..0000000000 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/ReplaceRequest.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.metron.indexing.dao.update; - -import com.fasterxml.jackson.databind.JsonNode; - -import java.util.Map; - -public class ReplaceRequest { - Map replacement; - String guid; - String sensorType; - String index; - - /** - * Return the index of the request. This is optional, but could result in better performance if specified. - * @return - */ - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - /** - * The sensor type of the request. This is mandatory. - * @return - */ - public String getSensorType() { - return sensorType; - } - - public void setSensorType(String sensorType) { - this.sensorType = sensorType; - } - - /** - * The replacement document. This is mandatory. - * @return - */ - public Map getReplacement() { - return replacement; - } - - public void setReplacement(Map replacement) { - this.replacement = replacement; - } - - /** - * The GUID of the document to replace. This is mandatory. - * @return - */ - public String getGuid() { - return guid; - } - - public void setGuid(String guid) { - this.guid = guid; - } -} diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java index 8f6f6b09af..82f0a4958b 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java @@ -17,11 +17,12 @@ */ package org.apache.metron.indexing.dao.update; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.indexing.dao.RetrieveLatestDao; + import java.io.IOException; import java.util.Map; import java.util.Optional; -import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.indexing.dao.RetrieveLatestDao; public interface UpdateDao { @@ -91,21 +92,4 @@ default Document getPatchedDocument(RetrieveLatestDao retrieveLatestDao, PatchRe request.getSensorType(), timestamp.orElse(System.currentTimeMillis())); } - - /** - * Replace a document in an index. - * @param request The replacement request. - * @param timestamp The timestamp (optional) of the update. If not specified, then current time will be used. - * @return The replaced document. - * @throws IOException If an error occurs during replacement. - */ - default Document replace(ReplaceRequest request, Optional timestamp) - throws IOException { - Document d = new Document(request.getReplacement(), - request.getGuid(), - request.getSensorType(), - timestamp.orElse(System.currentTimeMillis()) - ); - return update(d, Optional.ofNullable(request.getIndex())); - } } diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java index dbd3cb6b3c..5dbefddb6a 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java @@ -18,14 +18,6 @@ package org.apache.metron.indexing.dao; -import static org.apache.metron.common.Constants.SENSOR_TYPE; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.indexing.InMemoryMetaAlertRetrieveLatestDao; @@ -48,7 +40,15 @@ import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.ReplaceRequest; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import static org.apache.metron.common.Constants.SENSOR_TYPE; public class InMemoryMetaAlertDao implements MetaAlertDao { @@ -174,11 +174,6 @@ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request, return indexDao.patch(retrieveLatestDao, request, timestamp); } - @Override - public Document replace(ReplaceRequest request, Optional timestamp) throws IOException { - return indexDao.replace(request, timestamp); - } - @Override public SearchResponse getAllMetaAlertsForAlert(String guid) throws InvalidSearchException { SearchRequest request; diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java index ef9714edc1..b15895a93a 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java @@ -14,32 +14,30 @@ */ package org.apache.metron.indexing.dao; -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.stream.Collectors; import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.collections.MapUtils; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Result; import org.apache.metron.common.Constants; -import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.search.AlertComment; import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; +import static org.hamcrest.CoreMatchers.hasItem; + public abstract class UpdateIntegrationTest { /** @@ -62,215 +60,147 @@ public abstract class UpdateIntegrationTest { @Multiline protected String commentTwo; - private static final int MAX_RETRIES = 10; - private static final int SLEEP_MS = 500; + protected static final int MAX_RETRIES = 10; + protected static final int SLEEP_MS = 500; protected static final String SENSOR_NAME = "test"; private static final String CF = "p"; private MultiIndexDao dao; @Test - public void test() throws Exception { - List> inputData = new ArrayList<>(); - for(int i = 0; i < 10;++i) { - final String name = "message" + i; - inputData.add( - new HashMap() {{ - put("source.type", SENSOR_NAME); - put("name" , name); - put("timestamp", System.currentTimeMillis()); - put(Constants.GUID, name); - }} - ); - } - addTestData(getIndexName(), SENSOR_NAME, inputData); - List> docs = null; - for(int t = 0;t < MAX_RETRIES;++t, Thread.sleep(SLEEP_MS)) { - docs = getIndexedTestData(getIndexName(), SENSOR_NAME); - if(docs.size() >= 10) { - break; - } - } - Assert.assertEquals(10, docs.size()); - //modify the first message and add a new field + public void testUpdate() throws Exception { + // create a document to update + final String guid = UUID.randomUUID().toString(); + final Long timestamp = 1526306463050L; + Document toUpdate = createDocument(guid, timestamp); { - Map message0 = new HashMap(inputData.get(0)) {{ - put("new-field", "metron"); - }}; - String guid = "" + message0.get(Constants.GUID); - Document update = getDao().replace(new ReplaceRequest(){{ - setReplacement(message0); - setGuid(guid); - setSensorType(SENSOR_NAME); - setIndex(getIndexName()); - }}, Optional.empty()); - - Assert.assertEquals(message0, update.getDocument()); - Assert.assertEquals(1, getMockHTable().size()); - findUpdatedDoc(message0, guid, SENSOR_NAME); - { - //ensure hbase is up to date - Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME))); - Result r = getMockHTable().get(g); - NavigableMap columns = r.getFamilyMap(CF.getBytes()); - Assert.assertEquals(1, columns.size()); - Assert.assertEquals(message0 - , JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()) - , JSONUtils.MAP_SUPPLIER) - ); - } - { - //ensure ES is up-to-date - long cnt = 0; - for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { - docs = getIndexedTestData(getIndexName(), SENSOR_NAME); - cnt = docs - .stream() - .filter(d -> message0.get("new-field").equals(d.get("new-field"))) - .count(); - } - Assert.assertNotEquals("Data store is not updated!", cnt, 0); - } + // update the document and validate + Document updated = getDao().update(toUpdate, Optional.of(SENSOR_NAME)); + Assert.assertEquals(toUpdate, updated); } - //modify the same message and modify the new field { - Map message0 = new HashMap(inputData.get(0)) {{ - put("new-field", "metron2"); - }}; - String guid = "" + message0.get(Constants.GUID); - Document update = getDao().replace(new ReplaceRequest(){{ - setReplacement(message0); - setGuid(guid); - setSensorType(SENSOR_NAME); - setIndex(getIndexName()); - }}, Optional.empty()); - Assert.assertEquals(message0, update.getDocument()); - Assert.assertEquals(1, getMockHTable().size()); - Document doc = getDao().getLatest(guid, SENSOR_NAME); - Assert.assertEquals(message0, doc.getDocument()); - findUpdatedDoc(message0, guid, SENSOR_NAME); - { - //ensure hbase is up to date - Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME))); - Result r = getMockHTable().get(g); - NavigableMap columns = r.getFamilyMap(CF.getBytes()); - Assert.assertEquals(2, columns.size()); - Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue()) - , JSONUtils.MAP_SUPPLIER) - ); - Assert.assertNotEquals(message0, JSONUtils.INSTANCE.load(new String(columns.firstEntry().getValue()) - , JSONUtils.MAP_SUPPLIER) - ); - } - { - //ensure ES is up-to-date - long cnt = 0; - for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t,Thread.sleep(SLEEP_MS)) { - docs = getIndexedTestData(getIndexName(), SENSOR_NAME); - cnt = docs - .stream() - .filter(d -> message0.get("new-field").equals(d.get("new-field"))) - .count(); - } - - Assert.assertNotEquals("Data store is not updated!", cnt, 0); - } + // ensure the document is updated in the index + Document indexed = findUpdatedDoc(toUpdate.getDocument(), guid, SENSOR_NAME); + Assert.assertEquals(toUpdate, indexed); } } @Test - public void testAddCommentAndPatch() throws Exception { - Map fields = new HashMap<>(); - fields.put("guid", "add_comment"); - fields.put("source.type", SENSOR_NAME); + public void testBatchUpdate() throws Exception { + Map> toUpdate = new HashMap<>(); - Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526306463050L); - { - Document update = getDao().update(document, Optional.of(SENSOR_NAME)); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); - } - ArrayList comments = new ArrayList<>(); + // create the first document to update + final String guid1 = UUID.randomUUID().toString(); + final Long timestamp1 = 1526306463050L; + Document document1 = createDocument(guid1, timestamp1); + toUpdate.put(document1, Optional.of(SENSOR_NAME)); + + // create the second document to update + final String guid2 = UUID.randomUUID().toString(); + final Long timestamp2 = 1526306463100L; + Document document2 = createDocument(guid2, timestamp2); + toUpdate.put(document2, Optional.of(SENSOR_NAME)); + + // create the third document to update + final String guid3 = UUID.randomUUID().toString(); + final Long timestamp3 = 1526306463300L; + Document document3 = createDocument(guid3, timestamp3); + toUpdate.put(document3, Optional.of(SENSOR_NAME)); + + // update the documents as a batch and validate + Map> updated = getDao().batchUpdate(toUpdate); + Assert.assertThat(updated.keySet(), hasItem(document1)); + Assert.assertThat(updated.keySet(), hasItem(document2)); + Assert.assertThat(updated.keySet(), hasItem(document3)); + + // ensure the documents were written to the index + Assert.assertEquals(document1, findUpdatedDoc(document1.getDocument(), guid1, SENSOR_NAME)); + Assert.assertEquals(document2, findUpdatedDoc(document2.getDocument(), guid2, SENSOR_NAME)); + Assert.assertEquals(document3, findUpdatedDoc(document3.getDocument(), guid3, SENSOR_NAME)); + } + + @Test + public void testAddComment() throws Exception { + Document document = createAndIndexDocument("testAddCommentAndPatch"); + + // comment on the document + String commentText = "New Comment"; + String commentUser = "test_user"; + long commentTimestamp = 152630493050L; + Document withComment = addAlertComment(document.getGuid(), commentText, commentUser, commentTimestamp); { - Document update = addAlertComment("add_comment", "New Comment", "test_user", 1526306463050L); - // Ensure we have the first comment - comments.add(new AlertComment("New Comment", "test_user", 1526306463050L)); - document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( - Collectors.toList())); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + // validate that the comment was made on the returned document + List comments = getComments(withComment); + Assert.assertEquals(1, comments.size()); + Assert.assertEquals(commentText, comments.get(0).getComment()); + Assert.assertEquals(commentUser, comments.get(0).getUsername()); + Assert.assertEquals(commentTimestamp, comments.get(0).getTimestamp()); } { - List> patchList = new ArrayList<>(); - Map patch = new HashMap<>(); - patch.put("op", "add"); - patch.put("path", "/project"); - patch.put("value", "metron"); - patchList.add(patch); - - PatchRequest pr = new PatchRequest(); - pr.setGuid("add_comment"); - pr.setIndex(SENSOR_NAME); - pr.setSensorType(SENSOR_NAME); - pr.setPatch(patchList); - Document update = getDao().patch(getDao(), pr, Optional.of(1526306463050L)); - - document.getDocument().put("project", "metron"); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); + // validate that the comment was made on the indexed document + Document indexed = findUpdatedDoc(withComment.getDocument(), withComment.getGuid(), SENSOR_NAME); + List comments = getComments(indexed); + Assert.assertEquals(1, comments.size()); + Assert.assertEquals(commentText, comments.get(0).getComment()); + Assert.assertEquals(commentUser, comments.get(0).getUsername()); + Assert.assertEquals(commentTimestamp, comments.get(0).getTimestamp()); } } @Test - @SuppressWarnings("unchecked") + public void testPatchDocumentThatHasComment() throws Exception { + Document document = createAndIndexDocument("testPatchDocumentWithComment"); + + // comment on the document + String commentText = "New Comment"; + String commentUser = "test_user"; + long commentTimestamp = 152630493050L; + Document withComment = addAlertComment(document.getGuid(), commentText, commentUser, commentTimestamp); + + // create a patch + List> patches = new ArrayList<>(); + Map patch = new HashMap<>(); + patch.put("op", "add"); + patch.put("path", "/project"); + patch.put("value", "metron"); + patches.add(patch); + + PatchRequest pr = new PatchRequest(); + pr.setGuid(withComment.getGuid()); + pr.setIndex(SENSOR_NAME); + pr.setSensorType(SENSOR_NAME); + pr.setPatch(patches); + + // patch the document that has been commented on + Document patched = getDao().patch(getDao(), pr, Optional.of(withComment.getTimestamp())); + Assert.assertEquals("metron", patched.getDocument().get("project")); + + // ensure the patch was made on the indexed document + Document indexed = findUpdatedDoc(patched.getDocument(), patched.getGuid(), SENSOR_NAME); + Assert.assertEquals("metron", indexed.getDocument().get("project")); + } + + @Test public void testRemoveComments() throws Exception { - Map fields = new HashMap<>(); - fields.put("guid", "add_comment"); - fields.put("source.type", SENSOR_NAME); + String guid = "testRemoveComments"; + createAndIndexDocument(guid); - Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L); - { - Document update = getDao().update(document, Optional.of(SENSOR_NAME)); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); - } - ArrayList comments = new ArrayList<>(); - { - Document update = addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L); - // Ensure we have the first comment - - comments.add(new AlertComment("New Comment", "test_user", 1526401584951L)); - document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( - Collectors.toList())); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); - } - { - Document update = addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L); - // Ensure we have the second comment - comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L)); - document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( - Collectors.toList())); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); - } - { - Document update = removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L); - // Ensure we only have the first comments - comments = new ArrayList<>(); - comments.add(new AlertComment(commentOne)); - document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect( - Collectors.toList())); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); - } - { - Document update = removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L); - // Ensure we have no comments - document.getDocument().remove(COMMENTS_FIELD); - Assert.assertEquals(document, update); - findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME); - } + // add a comment on the document + Document withComments = addAlertComment(guid, "comment", "user1", 1526401584951L); + Assert.assertEquals(1, getComments(withComments).size()); + + // ensure the comment was added to the document in the index + Document indexedWithComments = findUpdatedDoc(withComments.getDocument(), withComments.getGuid(), withComments.getSensorType()); + Assert.assertEquals(1, getComments(indexedWithComments).size()); + + // remove a comment from the document + AlertComment toRemove = getComments(withComments).get(0); + Document noComments = removeAlertComment(guid, toRemove.getComment(), toRemove.getUsername(), toRemove.getTimestamp()); + Assert.assertEquals(0, getComments(noComments).size()); + + // ensure the comment was removed from the index + Document indexedNoComments = findUpdatedDoc(noComments.getDocument(), withComments.getGuid(), withComments.getSensorType()); + Assert.assertEquals(0, getComments(indexedNoComments).size()); } protected Document addAlertComment(String guid, String comment, String username, long timestamp) @@ -296,12 +226,47 @@ private CommentAddRemoveRequest buildAlertRequest(String guid, String comment, S return request; } - protected void findUpdatedDoc(Map message0, String guid, String sensorType) + private Document createAndIndexDocument(String guid) throws Exception { + // create the document + Long timestamp = 1526306463050L; + Document toCreate = createDocument(guid, timestamp); + + // index the document + Document created = getDao().update(toCreate, Optional.of(SENSOR_NAME)); + Assert.assertEquals(toCreate, created); + + // ensure the document is indexed + return findUpdatedDoc(toCreate.getDocument(), guid, SENSOR_NAME); + } + + protected Document createDocument(String guid, Long timestamp) { + Map message1 = new HashMap<>(); + message1.put(Constants.GUID, guid); + message1.put(Constants.SENSOR_TYPE, SENSOR_NAME); + message1.put(Constants.Fields.TIMESTAMP.getName(), timestamp); + + return new Document(message1, guid, SENSOR_NAME, timestamp); + } + + private List getComments(Document withComment) { + List> commentsField = List.class.cast(withComment.getDocument().get(COMMENTS_FIELD)); + List comments = new ArrayList<>(); + if(commentsField != null) { + comments = commentsField + .stream() + .map(map -> new AlertComment(map)) + .collect(Collectors.toList()); + } + + return comments; + } + + protected Document findUpdatedDoc(Map message0, String guid, String sensorType) throws InterruptedException, IOException, OriginalNotFoundException { for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { Document doc = getDao().getLatest(guid, sensorType); if (doc != null && message0.equals(doc.getDocument())) { - return; + return doc; } if (t == MAX_RETRIES -1) { MapUtils.debugPrint(System.out, "Expected", message0); diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java index 8eaa8f47ef..831b20bf09 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java @@ -18,18 +18,6 @@ package org.apache.metron.indexing.integration; -import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF; -import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE; -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.AccessConfig; @@ -44,6 +32,19 @@ import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF; +import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + public class HBaseDaoIntegrationTest extends UpdateIntegrationTest { private static final String TABLE_NAME = "metron_update"; @@ -169,13 +170,6 @@ public void shouldGetAllLatest() throws Exception { Assert.assertFalse("Result size should be 12 but was greater", results.hasNext()); } - @Override - public void test() { - // The main test ensures a variety of things not implemented by HBase run alongside - // HBaseDao itself. - // Therefore, just don't do anything for this test. - } - protected List buildAlerts(int count) throws IOException { List alerts = new ArrayList<>(); for (int i = 0; i < count; ++i) { diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java index e6906f311c..133a79668c 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -17,14 +17,7 @@ */ package org.apache.metron.solr.dao; -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; - import com.google.common.base.Splitter; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; -import java.util.Map; -import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.ColumnMetadataDao; import org.apache.metron.indexing.dao.IndexDao; @@ -47,6 +40,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; + public class SolrDao implements IndexDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java index 8ef9484a5e..a1fef97898 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java @@ -18,13 +18,6 @@ package org.apache.metron.solr.dao; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; - import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -49,6 +42,13 @@ import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.solr.client.solrj.SolrClient; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + public class SolrMetaAlertDao implements MetaAlertDao { public static final String METAALERTS_COLLECTION = "metaalert"; diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java index 54b5b64f33..9051f89e7e 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java @@ -17,7 +17,16 @@ */ package org.apache.metron.solr.dao; -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.UpdateDao; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -31,16 +40,8 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.apache.metron.indexing.dao.AccessConfig; -import org.apache.metron.indexing.dao.search.AlertComment; -import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; -import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.UpdateDao; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; public class SolrUpdateDao implements UpdateDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java index 8920d5a94c..b761f36b43 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java @@ -18,14 +18,6 @@ package org.apache.metron.solr.dao; -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.HBaseDao; import org.apache.metron.indexing.dao.IndexDao; @@ -45,6 +37,15 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; + public class SolrMetaAlertDaoTest { private static AccessConfig accessConfig = new AccessConfig(); diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index 5b965590c7..0b7b0e9cba 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -17,14 +17,6 @@ */ package org.apache.metron.solr.integration; -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; @@ -50,6 +42,15 @@ import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; +import static org.junit.Assert.assertEquals; + public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { @Rule public final ExpectedException exception = ExpectedException.none(); From 428a12f5ad883c0aa19e1c9040dee2070a97a6a4 Mon Sep 17 00:00:00 2001 From: Nick Allen Date: Thu, 29 Nov 2018 10:51:12 -0500 Subject: [PATCH 2/2] Trying to keep the changes as focused on remove the ReplaceRequest as possible --- .../apache/metron/indexing/dao/HBaseDao.java | 23 ++++++++------- .../metron/indexing/dao/MultiIndexDao.java | 17 ++++++----- .../AbstractLuceneMetaAlertUpdateDao.java | 28 +++++++++---------- .../integration/HBaseDaoIntegrationTest.java | 25 ++++++++--------- .../org/apache/metron/solr/dao/SolrDao.java | 15 +++++----- .../metron/solr/dao/SolrMetaAlertDao.java | 14 +++++----- .../apache/metron/solr/dao/SolrUpdateDao.java | 23 ++++++++------- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 17 ++++++----- .../SolrUpdateIntegrationTest.java | 17 ++++++----- 9 files changed, 86 insertions(+), 93 deletions(-) diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java index 9bdb3ceec1..71d05444c9 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java @@ -18,6 +18,17 @@ package org.apache.metron.indexing.dao; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; @@ -37,18 +48,6 @@ import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Optional; -import java.util.stream.Collectors; - /** * The HBaseDao is an index dao which only supports the following actions: * * Update diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java index f41a82f0ae..c3e2108e99 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java @@ -20,6 +20,14 @@ import com.google.common.base.Joiner; import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; @@ -31,15 +39,6 @@ import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; import org.apache.metron.indexing.dao.update.Document; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - public class MultiIndexDao implements IndexDao { private List indices; diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java index 39396f05ad..096baf174b 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java @@ -18,19 +18,7 @@ package org.apache.metron.indexing.dao.metaalert.lucene; -import org.apache.metron.common.Constants; -import org.apache.metron.indexing.dao.RetrieveLatestDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; -import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; -import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; -import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; -import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao; -import org.apache.metron.indexing.dao.metaalert.MetaScores; -import org.apache.metron.indexing.dao.search.GetRequest; -import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.OriginalNotFoundException; -import org.apache.metron.indexing.dao.update.PatchRequest; -import org.apache.metron.indexing.dao.update.UpdateDao; +import static org.apache.metron.common.Constants.GUID; import java.io.IOException; import java.util.ArrayList; @@ -46,7 +34,19 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import static org.apache.metron.common.Constants.GUID; +import org.apache.metron.common.Constants; +import org.apache.metron.indexing.dao.RetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig; +import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants; +import org.apache.metron.indexing.dao.metaalert.MetaAlertRetrieveLatestDao; +import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus; +import org.apache.metron.indexing.dao.metaalert.MetaAlertUpdateDao; +import org.apache.metron.indexing.dao.metaalert.MetaScores; +import org.apache.metron.indexing.dao.search.GetRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.OriginalNotFoundException; +import org.apache.metron.indexing.dao.update.PatchRequest; +import org.apache.metron.indexing.dao.update.UpdateDao; public abstract class AbstractLuceneMetaAlertUpdateDao implements MetaAlertUpdateDao { diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java index 831b20bf09..8148b6966e 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java @@ -18,6 +18,18 @@ package org.apache.metron.indexing.integration; +import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF; +import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; import org.apache.metron.hbase.mock.MockHBaseTableProvider; import org.apache.metron.hbase.mock.MockHTable; import org.apache.metron.indexing.dao.AccessConfig; @@ -32,19 +44,6 @@ import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF; -import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE; -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; - public class HBaseDaoIntegrationTest extends UpdateIntegrationTest { private static final String TABLE_NAME = "metron_update"; diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java index 133a79668c..e6906f311c 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java @@ -17,7 +17,14 @@ */ package org.apache.metron.solr.dao; +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; + import com.google.common.base.Splitter; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.ColumnMetadataDao; import org.apache.metron.indexing.dao.IndexDao; @@ -40,14 +47,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; - public class SolrDao implements IndexDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java index a1fef97898..8ef9484a5e 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrMetaAlertDao.java @@ -18,6 +18,13 @@ package org.apache.metron.solr.dao; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + import org.apache.metron.common.Constants; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; @@ -42,13 +49,6 @@ import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.solr.client.solrj.SolrClient; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; - public class SolrMetaAlertDao implements MetaAlertDao { public static final String METAALERTS_COLLECTION = "metaalert"; diff --git a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java index 9051f89e7e..54b5b64f33 100644 --- a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java +++ b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java @@ -17,16 +17,7 @@ */ package org.apache.metron.solr.dao; -import org.apache.metron.indexing.dao.AccessConfig; -import org.apache.metron.indexing.dao.search.AlertComment; -import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; -import org.apache.metron.indexing.dao.update.Document; -import org.apache.metron.indexing.dao.update.UpdateDao; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -40,8 +31,16 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - -import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD; +import org.apache.metron.indexing.dao.AccessConfig; +import org.apache.metron.indexing.dao.search.AlertComment; +import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest; +import org.apache.metron.indexing.dao.update.Document; +import org.apache.metron.indexing.dao.update.UpdateDao; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SolrUpdateDao implements UpdateDao { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java index b761f36b43..8920d5a94c 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java @@ -18,6 +18,14 @@ package org.apache.metron.solr.dao; +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.HBaseDao; import org.apache.metron.indexing.dao.IndexDao; @@ -37,15 +45,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; - public class SolrMetaAlertDaoTest { private static AccessConfig accessConfig = new AccessConfig(); diff --git a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java index 0b7b0e9cba..5b965590c7 100644 --- a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java +++ b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/integration/SolrUpdateIntegrationTest.java @@ -17,6 +17,14 @@ */ package org.apache.metron.solr.integration; +import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.conf.Configuration; @@ -42,15 +50,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER; -import static org.junit.Assert.assertEquals; - public class SolrUpdateIntegrationTest extends UpdateIntegrationTest { @Rule public final ExpectedException exception = ExpectedException.none();