From 38194c7fd554ed939d340d060f0e4e71cf7e251a Mon Sep 17 00:00:00 2001 From: Jason Han Date: Fri, 5 Feb 2021 16:29:21 -0800 Subject: [PATCH 01/11] Remove reference to sdvc --- .../openmbee/mms/core/dao/NodeIndexDAO.java | 10 ++- .../mms/elastic/BaseElasticDAOImpl.java | 73 ++++++++++--------- .../mms/elastic/BranchElasticDAOImpl.java | 11 +-- .../mms/elastic/CommitElasticDAOImpl.java | 13 ++-- .../mms/elastic/NodeElasticDAOImpl.java | 9 ++- 5 files changed, 61 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java b/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java index e30ddea8b..4004dc260 100644 --- a/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java +++ b/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java @@ -4,22 +4,24 @@ import java.util.List; import java.util.Optional; import java.util.Set; + +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.ElementJson; public interface NodeIndexDAO { - void indexAll(Collection jsons); + void indexAll(Collection jsons) throws MMSException; - void index(BaseJson json); + void index(BaseJson json) throws MMSException; Optional findById(String docId); List findAllById(Set docIds); - void deleteById(String docId); + void deleteById(String docId) throws MMSException; - void deleteAll(Collection jsons); + void deleteAll(Collection jsons) throws MMSException; boolean existsById(String docId); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index f75de49bb..e5b0334fe 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.openmbee.mms.core.exceptions.InternalErrorException; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.slf4j.Logger; @@ -79,7 +80,7 @@ public void deleteById(String index, String docId) { try { client.delete(new DeleteRequest(index, docId), REQUEST_OPTIONS); } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } } @@ -91,7 +92,7 @@ public void deleteAll(String index, Collection jsons) { } client.bulk(bulkIndex, REQUEST_OPTIONS); } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } } @@ -102,7 +103,7 @@ public boolean existsById(String index, String docId) { getRequest.storedFields("_none_"); return client.exists(getRequest, REQUEST_OPTIONS); } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } } @@ -118,7 +119,7 @@ public Optional findById(String index, String docId) { return Optional.empty(); } } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } } @@ -145,7 +146,7 @@ public List findAllById(String index, Set docIds) { } return listOfResponses; } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } } @@ -172,9 +173,11 @@ public void indexAll(String index, Collection jsons) { try { if (!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) { logger.error("Timed out in bulk processing"); + throw new InternalErrorException("Timeout error in bulk processing"); } } catch (InterruptedException e) { logger.error("Index all interrupted: ", e); + throw new InternalErrorException(e); } } @@ -183,7 +186,7 @@ public void index(String index, BaseJson json) { try { client.index(new IndexRequest(index).id(json.getDocId()).source(json), REQUEST_OPTIONS); } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } } @@ -203,42 +206,14 @@ public E update(String index, BaseJson json) { } } } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalErrorException(e); } return response; } protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { - return getBulkProcessor(client, null); - } - - private BulkProcessor getBulkProcessor(RestHighLevelClient client, BulkProcessor.Listener listener) { - if (listener == null) { - listener = new BulkProcessor.Listener() { - private final Logger logger = LoggerFactory.getLogger(getClass()); - @Override - public void beforeBulk(long executionId, BulkRequest request) { - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - if (response.hasFailures()) { - response.iterator().forEachRemaining(action -> { - if (action.isFailed()) { - logger.error("Error in bulk processing: {}", action.getFailureMessage()); - } - }); - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - logger.error("Error in bulk processing: ", failure); - } - }; - } BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> client - .bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); + .bulkAsync(request, RequestOptions.DEFAULT, bulkListener), getListener()); bpBuilder.setBulkActions(bulkLimit); bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)); bpBuilder.setConcurrentRequests(1); @@ -247,4 +222,30 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) return bpBuilder.build(); } + + private BulkProcessor.Listener getListener() { + return new BulkProcessor.Listener() { + private final Logger logger = LoggerFactory.getLogger(getClass()); + @Override + public void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + if (response.hasFailures()) { + response.iterator().forEachRemaining(action -> { + if (action.isFailed()) { + logger.error("Error in bulk processing: {}", action.getFailureMessage()); + } + }); + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Error in bulk processing: ", failure); + throw new InternalErrorException(failure); + } + }; + } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java index 83ccb3e9b..7592fbabe 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java @@ -7,6 +7,7 @@ import java.util.UUID; import org.openmbee.mms.core.dao.BranchIndexDAO; +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.RefJson; import org.springframework.stereotype.Component; @@ -19,12 +20,12 @@ protected RefJson newInstance() { } @Override - public void indexAll(Collection jsons) { + public void indexAll(Collection jsons) throws MMSException { this.indexAll(getIndex(), jsons); } @Override - public void index(BaseJson json) { + public void index(BaseJson json) throws MMSException { this.index(getIndex(), json); } @@ -36,11 +37,11 @@ public List findAllById(Set docIds) { return this.findAllById(getIndex(), docIds); } - public void deleteById(String docId) { + public void deleteById(String docId) throws MMSException { this.deleteById(getIndex(), docId); } - public void deleteAll(Collection jsons) { + public void deleteAll(Collection jsons) throws MMSException { this.deleteAll(getIndex(), jsons); } @@ -49,7 +50,7 @@ public boolean existsById(String docId) { } @Override - public RefJson update(RefJson refJson) { + public RefJson update(RefJson refJson) throws MMSException { return this.update(getIndex(), refJson); } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index d22dbbae0..28d370e11 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.openmbee.mms.core.dao.CommitIndexDAO; +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.CommitJson; @@ -31,15 +32,15 @@ protected CommitJson newInstance() { return new CommitJson(); } - public void indexAll(Collection jsons) { + public void indexAll(Collection jsons) throws MMSException { this.indexAll(getIndex(), jsons); } - public void index(BaseJson json) { + public void index(BaseJson json) throws MMSException { index((CommitJson) json); } - public void index(CommitJson json) { + public void index(CommitJson json) throws MMSException { int commitCount = getCommitSize(json); List broken = new ArrayList<>(); if (commitCount > commitLimit) { @@ -88,14 +89,14 @@ public List findAllById(Set commitIds) { return getFullCommits(commitIds); } - public void deleteById(String commitId) { + public void deleteById(String commitId) throws MMSException { List docs = getDocs(commitId); docs.forEach(commit -> { this.deleteById(getIndex(), commit.getDocId()); }); } - public void deleteAll(Collection jsons) { + public void deleteAll(Collection jsons) throws MMSException { this.deleteAll(getIndex(), jsons); } @@ -166,7 +167,7 @@ protected String getIndex() { } @Override - public CommitJson update(CommitJson commitJson) { + public CommitJson update(CommitJson commitJson) throws MMSException { return this.update(getIndex(), commitJson); } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index 2804f20b8..0496836a5 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -23,6 +23,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.dao.NodeIndexDAO; +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.ElementJson; @@ -35,11 +36,11 @@ protected ElementJson newInstance() { return new ElementJson(); } - public void indexAll(Collection jsons) { + public void indexAll(Collection jsons) throws MMSException { this.indexAll(getIndex(), jsons); } - public void index(BaseJson json) { + public void index(BaseJson json) throws MMSException { this.index(getIndex(), json); } @@ -51,11 +52,11 @@ public List findAllById(Set docIds) { return this.findAllById(getIndex(), docIds); } - public void deleteById(String docId) { + public void deleteById(String docId) throws MMSException { this.deleteById(getIndex(), docId); } - public void deleteAll(Collection jsons) { + public void deleteAll(Collection jsons) throws MMSException { this.deleteAll(getIndex(), jsons); } From 5ff660ff33f007344c900c4f83dd21c14ec183d3 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 9 Feb 2021 09:30:18 -0800 Subject: [PATCH 02/11] Use synchronous bulk processing instead --- .../mms/crud/services/DefaultNodeService.java | 24 ++++++++----- .../mms/elastic/BaseElasticDAOImpl.java | 36 ++++++++----------- .../mms/elastic/NodeElasticDAOImpl.java | 8 +---- .../mms/rdb/repositories/BaseDAOImpl.java | 2 -- .../rdb/repositories/node/NodeDAOImpl.java | 2 +- 5 files changed, 32 insertions(+), 40 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 772825d54..bdbb105f8 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -22,7 +22,6 @@ import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.objects.ElementsRequest; import org.openmbee.mms.core.objects.ElementsResponse; -import org.openmbee.mms.core.exceptions.InternalErrorException; import org.openmbee.mms.data.domains.scoped.Commit; import org.openmbee.mms.data.domains.scoped.CommitType; import org.openmbee.mms.data.domains.scoped.Node; @@ -37,12 +36,16 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.DefaultTransactionDefinition; @Service("defaultNodeService") +@EnableTransactionManagement public class DefaultNodeService implements NodeService { @Value("${mms.stream.batch.size:100000}") @@ -62,6 +65,8 @@ public class DefaultNodeService implements NodeService { protected Collection eventPublisher; + protected PlatformTransactionManager platformTransactionManager; + @Autowired public void setNodeRepository(NodeDAO nodeRepository) { this.nodeRepository = nodeRepository; @@ -102,6 +107,11 @@ public void setEventPublisher(Collection eventPublisher) { this.eventPublisher = eventPublisher; } + @Autowired + public void setPlatformTransactionManager(PlatformTransactionManager platformTransactionManager) { + this.platformTransactionManager = platformTransactionManager; + } + @Override public void readAsStream(String projectId, String refId, Map params, OutputStream stream, String accept) throws IOException { @@ -144,6 +154,7 @@ public void readAsStream(String projectId, String refId, } else { stream.write("\n".getBytes(StandardCharsets.UTF_8)); } + stream.close(); } @Override @@ -207,8 +218,8 @@ public ElementsCommitResponse createOrUpdate(String projectId, String refId, Ele return response; } - //@Transactional - protected void commitChanges(NodeChangeInfo info) { + @Transactional + public void commitChanges(NodeChangeInfo info) { //TODO: Test rollback on IndexDAO failure TransactionDefinition def = new DefaultTransactionDefinition(); TransactionStatus status = this.nodeRepository.getTransactionManager().getTransaction(def); @@ -219,7 +230,6 @@ protected void commitChanges(NodeChangeInfo info) { Instant now = info.getNow(); if (!nodes.isEmpty()) { try { - this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); if (json != null && !json.isEmpty()) { this.nodeIndex.indexAll(json.values()); } @@ -241,12 +251,10 @@ protected void commitChanges(NodeChangeInfo info) { this.commitRepository.save(commit); }); this.commitIndex.index(cmjs); - + this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); this.nodeRepository.getTransactionManager().commit(status); } catch (Exception e) { - logger.error("commitChanges error: ", e); - this.nodeRepository.getTransactionManager().rollback(status); - throw new InternalErrorException("Error committing transaction"); + logger.error("commitChanges error: {}", e.getMessage()); } eventPublisher.forEach((pub) -> pub.publish( EventObject.create(cmjs.getProjectId(), cmjs.getRefId(), "commit", cmjs))); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index e5b0334fe..4d1b42c15 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkProcessor; @@ -170,16 +169,7 @@ public void indexAll(String index, Collection jsons) { for (BaseJson json : jsons) { bulkProcessor.add(new IndexRequest(index).id(json.getDocId()).source(json)); } - try { - if (!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) { - logger.error("Timed out in bulk processing"); - throw new InternalErrorException("Timeout error in bulk processing"); - } - } catch (InterruptedException e) { - logger.error("Index all interrupted: ", e); - throw new InternalErrorException(e); - } - + bulkProcessor.close(); } public void index(String index, BaseJson json) { @@ -212,8 +202,18 @@ public E update(String index, BaseJson json) { } protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { - BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> client - .bulkAsync(request, RequestOptions.DEFAULT, bulkListener), getListener()); + BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> { + try { + BulkResponse response = client.bulk(request, RequestOptions.DEFAULT); + if (response.hasFailures()) { + String failure = response.buildFailureMessage(); + logger.error("Bulk response error: {}", failure); + throw new InternalErrorException(failure); + } + } catch (IOException ioe) { + throw new InternalErrorException(ioe); + } + }, getListener()); bpBuilder.setBulkActions(bulkLimit); bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)); bpBuilder.setConcurrentRequests(1); @@ -225,25 +225,17 @@ protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { private BulkProcessor.Listener getListener() { return new BulkProcessor.Listener() { - private final Logger logger = LoggerFactory.getLogger(getClass()); @Override public void beforeBulk(long executionId, BulkRequest request) { } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - if (response.hasFailures()) { - response.iterator().forEachRemaining(action -> { - if (action.isFailed()) { - logger.error("Error in bulk processing: {}", action.getFailureMessage()); - } - }); - } + } @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - logger.error("Error in bulk processing: ", failure); throw new InternalErrorException(failure); } }; diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index 0496836a5..ae196372f 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -113,13 +113,7 @@ private void bulkUpdateRefWithScript(Set docIds, String script) { request.script(inline); bulkProcessor.add(request); } - try { - if (!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) { - logger.error("Timed out in bulk processing"); - } - } catch (InterruptedException e) { - logger.error("Index all interrupted: ", e); - } + bulkProcessor.close(); } @Override diff --git a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/BaseDAOImpl.java b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/BaseDAOImpl.java index 4a5e21365..d2f878092 100644 --- a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/BaseDAOImpl.java +++ b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/BaseDAOImpl.java @@ -9,8 +9,6 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.PlatformTransactionManager; -import java.util.regex.Pattern; - import static org.openmbee.mms.core.config.Constants.BRANCH_ID_VALID_PATTERN; public abstract class BaseDAOImpl { diff --git a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/node/NodeDAOImpl.java b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/node/NodeDAOImpl.java index 4ed518e6e..d40e9ef58 100644 --- a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/node/NodeDAOImpl.java +++ b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/node/NodeDAOImpl.java @@ -193,4 +193,4 @@ private PreparedStatement prepareStatement(PreparedStatement ps, Node n) throws } return ps; } -} +} \ No newline at end of file From 9c814c7f884e43be24bb86cc8351accc0561f57b Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 9 Feb 2021 09:41:52 -0800 Subject: [PATCH 03/11] Clean up runtime exceptions --- .../java/org/openmbee/mms/core/dao/NodeIndexDAO.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java b/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java index 4004dc260..8d14d5fed 100644 --- a/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java +++ b/core/src/main/java/org/openmbee/mms/core/dao/NodeIndexDAO.java @@ -5,23 +5,22 @@ import java.util.Optional; import java.util.Set; -import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.ElementJson; public interface NodeIndexDAO { - void indexAll(Collection jsons) throws MMSException; + void indexAll(Collection jsons); - void index(BaseJson json) throws MMSException; + void index(BaseJson json); Optional findById(String docId); List findAllById(Set docIds); - void deleteById(String docId) throws MMSException; + void deleteById(String docId); - void deleteAll(Collection jsons) throws MMSException; + void deleteAll(Collection jsons); boolean existsById(String docId); From 0db8d0e9506b64b6b19f6c18d4ef43d755bc9a8e Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 9 Feb 2021 10:32:18 -0800 Subject: [PATCH 04/11] Clean up more runtime exceptions --- .../openmbee/mms/elastic/CommitElasticDAOImpl.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index 28d370e11..d22dbbae0 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -15,7 +15,6 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.openmbee.mms.core.dao.CommitIndexDAO; -import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.CommitJson; @@ -32,15 +31,15 @@ protected CommitJson newInstance() { return new CommitJson(); } - public void indexAll(Collection jsons) throws MMSException { + public void indexAll(Collection jsons) { this.indexAll(getIndex(), jsons); } - public void index(BaseJson json) throws MMSException { + public void index(BaseJson json) { index((CommitJson) json); } - public void index(CommitJson json) throws MMSException { + public void index(CommitJson json) { int commitCount = getCommitSize(json); List broken = new ArrayList<>(); if (commitCount > commitLimit) { @@ -89,14 +88,14 @@ public List findAllById(Set commitIds) { return getFullCommits(commitIds); } - public void deleteById(String commitId) throws MMSException { + public void deleteById(String commitId) { List docs = getDocs(commitId); docs.forEach(commit -> { this.deleteById(getIndex(), commit.getDocId()); }); } - public void deleteAll(Collection jsons) throws MMSException { + public void deleteAll(Collection jsons) { this.deleteAll(getIndex(), jsons); } @@ -167,7 +166,7 @@ protected String getIndex() { } @Override - public CommitJson update(CommitJson commitJson) throws MMSException { + public CommitJson update(CommitJson commitJson) { return this.update(getIndex(), commitJson); } From 84e995302f52eb6a8f6c68f20c1073eb02ca6b9e Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Tue, 9 Feb 2021 12:54:06 -0800 Subject: [PATCH 05/11] add elastic specific tests --- .circleci/config.yml | 1 + example/elastic.postman_collection.json | 461 ++++++++++++++++++++++++ 2 files changed, 462 insertions(+) create mode 100644 example/elastic.postman_collection.json diff --git a/.circleci/config.yml b/.circleci/config.yml index c8f9dc3ac..6688b6c9b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,6 +38,7 @@ jobs: docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 500 docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run search.postman_collection.json -e test-env.json --delay-request 1000 docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 500 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run elastic.postman_collection.json -e test-env.json --delay-request 500 - persist_to_workspace: root: /home/circleci/ diff --git a/example/elastic.postman_collection.json b/example/elastic.postman_collection.json new file mode 100644 index 000000000..30463bfa2 --- /dev/null +++ b/example/elastic.postman_collection.json @@ -0,0 +1,461 @@ +{ + "info": { + "_postman_id": "41251e24-17eb-4133-8e98-edbaa90de5c2", + "name": "elastic", + "description": "elastic-fail tests", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "login using admin", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "", + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + " ", + "});", + "", + "pm.test(\"response has token\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.token).to.be.a('string');", + " pm.environment.set(\"token\", jsonData.token);", + "", + "});", + "", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [ + { + "key": "Content-Type", + "name": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"username\": \"{{adminUsername}}\",\n\t\"password\": \"{{adminPassword}}\"\n}" + }, + "url": { + "raw": "{{host}}/authentication", + "host": [ + "{{host}}" + ], + "path": [ + "authentication" + ] + } + }, + "response": [] + }, + { + "name": "add org elastic", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has org elastic\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.orgs[0].id).to.eql('elastic');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n\t\"orgs\": [\n\t\t{\n\t\t\t\"id\": \"elastic\",\n\t\t\t\"name\": \"elastic\"\n\t\t}\n\t]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/orgs", + "host": [ + "{{host}}" + ], + "path": [ + "orgs" + ] + } + }, + "response": [] + }, + { + "name": "add project elastic", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has project elastic\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.projects[0].id).to.eql('elastic');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"projects\": [\n\t\t{\n\t\t\t\"id\": \"elastic\", \n\t\t\t\"name\": \"elastic\",\n\t\t\t\"orgId\": \"elastic\",\n\t\t\t\"schema\": \"default\"\n\t\t}\n\t]\n}" + }, + "url": { + "raw": "{{host}}/projects", + "host": [ + "{{host}}" + ], + "path": [ + "projects" + ] + } + }, + "response": [] + }, + { + "name": "add element to elastic", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has element\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.elements.length).to.eql(1);", + "});", + "", + "pm.environment.set(\"commitId1\", pm.response.json().elements[0]._commitId);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": \"x\"\n\t\t}\n\t]\n}" + }, + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements" + ] + } + }, + "response": [] + }, + { + "name": "update bad element to elastic", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 500\", function () {", + " pm.response.to.have.status(500);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": {}\n\t\t}\n\t]\n}" + }, + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements" + ] + } + }, + "response": [] + }, + { + "name": "get x", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "pm.test(\"response has element x\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.elements[0].name).to.eql('x');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements/x", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements", + "x" + ] + } + }, + "response": [] + }, + { + "name": "update element to elastic", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.environment.set(\"commitId2\", pm.response.json().elements[0]._commitId);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": \"y\"\n\t\t}\n\t]\n}" + }, + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements" + ] + } + }, + "response": [] + }, + { + "name": "get x again", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "pm.test(\"response has element x\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.elements[0].name).to.eql('y');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements/x", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements", + "x" + ] + } + }, + "response": [] + }, + { + "name": "check ref commit history", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has 2 commit\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.commits.length).to.eql(2);", + "});", + "", + "pm.test(\"commit history is right\", function() {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId2\"))", + " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", + "});", + "", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/projects/elastic/refs/master/commits", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "commits" + ] + } + }, + "response": [] + }, + { + "name": "check element commit history", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has 2 commit\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.commits.length).to.eql(2);", + "});", + "", + "pm.test(\"commit history is right\", function() {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId2\"))", + " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", + "});", + "", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements/x/commits", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements", + "x", + "commits" + ] + } + }, + "response": [] + } + ] +} \ No newline at end of file From d9adb0a2e66caa25bb297fc5ac4bac9af19c1b49 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Tue, 9 Feb 2021 12:57:21 -0800 Subject: [PATCH 06/11] fix test header --- example/elastic.postman_collection.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/example/elastic.postman_collection.json b/example/elastic.postman_collection.json index 30463bfa2..992d78747 100644 --- a/example/elastic.postman_collection.json +++ b/example/elastic.postman_collection.json @@ -79,7 +79,13 @@ ], "request": { "method": "POST", - "header": [], + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", "raw": "{\n\t\"orgs\": [\n\t\t{\n\t\t\t\"id\": \"elastic\",\n\t\t\t\"name\": \"elastic\"\n\t\t}\n\t]\n}", From 935a7ee10fb8b7824900c612069700c68b67f194 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Tue, 9 Feb 2021 13:11:05 -0800 Subject: [PATCH 07/11] fix test auth --- example/elastic.postman_collection.json | 42 +++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/example/elastic.postman_collection.json b/example/elastic.postman_collection.json index 992d78747..992bfb2e1 100644 --- a/example/elastic.postman_collection.json +++ b/example/elastic.postman_collection.json @@ -80,12 +80,12 @@ "request": { "method": "POST", "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], "body": { "mode": "raw", "raw": "{\n\t\"orgs\": [\n\t\t{\n\t\t\t\"id\": \"elastic\",\n\t\t\t\"name\": \"elastic\"\n\t\t}\n\t]\n}", @@ -463,5 +463,35 @@ }, "response": [] } + ], + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{token}}", + "type": "string" + } + ] + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } ] } \ No newline at end of file From 9809a613f2190f0522b4ef2cf81871827164e307 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 9 Feb 2021 14:58:42 -0800 Subject: [PATCH 08/11] Clean up left over transaction code and reorder indexing --- .../mms/crud/services/DefaultNodeService.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index bdbb105f8..fce4485ab 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -36,16 +36,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.DefaultTransactionDefinition; - @Service("defaultNodeService") -@EnableTransactionManagement public class DefaultNodeService implements NodeService { @Value("${mms.stream.batch.size:100000}") @@ -65,8 +57,6 @@ public class DefaultNodeService implements NodeService { protected Collection eventPublisher; - protected PlatformTransactionManager platformTransactionManager; - @Autowired public void setNodeRepository(NodeDAO nodeRepository) { this.nodeRepository = nodeRepository; @@ -107,11 +97,6 @@ public void setEventPublisher(Collection eventPublisher) { this.eventPublisher = eventPublisher; } - @Autowired - public void setPlatformTransactionManager(PlatformTransactionManager platformTransactionManager) { - this.platformTransactionManager = platformTransactionManager; - } - @Override public void readAsStream(String projectId, String refId, Map params, OutputStream stream, String accept) throws IOException { @@ -218,12 +203,7 @@ public ElementsCommitResponse createOrUpdate(String projectId, String refId, Ele return response; } - @Transactional public void commitChanges(NodeChangeInfo info) { - //TODO: Test rollback on IndexDAO failure - TransactionDefinition def = new DefaultTransactionDefinition(); - TransactionStatus status = this.nodeRepository.getTransactionManager().getTransaction(def); - Map nodes = info.getToSaveNodeMap(); Map json = info.getUpdatedMap(); CommitJson cmjs = info.getCommitJson(); @@ -234,6 +214,7 @@ public void commitChanges(NodeChangeInfo info) { this.nodeIndex.indexAll(json.values()); } this.nodeIndex.removeFromRef(info.getOldDocIds()); + this.commitIndex.index(cmjs); Optional existing = this.commitRepository.findByCommitId(cmjs.getId()); existing.ifPresentOrElse( @@ -250,9 +231,7 @@ public void commitChanges(NodeChangeInfo info) { commit.setComment(cmjs.getComment()); this.commitRepository.save(commit); }); - this.commitIndex.index(cmjs); this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); - this.nodeRepository.getTransactionManager().commit(status); } catch (Exception e) { logger.error("commitChanges error: {}", e.getMessage()); } From e93c879f639bd7ade3dca69a94a4fb90a738262d Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 9 Feb 2021 15:27:14 -0800 Subject: [PATCH 09/11] Throw exception on rejected commit --- .../java/org/openmbee/mms/crud/services/DefaultNodeService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index fce4485ab..5c6748fc1 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -13,6 +13,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.openmbee.mms.core.exceptions.BadRequestException; +import org.openmbee.mms.core.exceptions.InternalErrorException; import org.openmbee.mms.core.objects.ElementsCommitResponse; import org.openmbee.mms.core.objects.EventObject; import org.openmbee.mms.core.services.EventService; @@ -234,6 +235,7 @@ public void commitChanges(NodeChangeInfo info) { this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); } catch (Exception e) { logger.error("commitChanges error: {}", e.getMessage()); + throw new InternalErrorException("Error committing changes."); } eventPublisher.forEach((pub) -> pub.publish( EventObject.create(cmjs.getProjectId(), cmjs.getRefId(), "commit", cmjs))); From c011b37a657085bf8436fad7ac699f56f46e3767 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Tue, 9 Feb 2021 18:04:35 -0800 Subject: [PATCH 10/11] log more stuff --- .../mms/core/exceptions/MMSException.java | 15 +++ .../crud/services/DefaultBranchService.java | 2 +- .../mms/crud/services/DefaultNodeService.java | 4 +- .../mms/elastic/BaseElasticDAOImpl.java | 8 ++ .../mms/elastic/BranchElasticDAOImpl.java | 10 +- .../mms/elastic/CommitElasticDAOImpl.java | 7 +- .../mms/elastic/NodeElasticDAOImpl.java | 17 +-- .../mms/elastic/ProjectElasticImpl.java | 7 +- .../services/ElasticSearchService.java | 3 +- example/elastic.postman_collection.json | 100 +++++++++++++++++- 10 files changed, 147 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java index bc45a655f..cdede62dc 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java @@ -11,11 +11,18 @@ public MMSException() { } public MMSException(HttpStatus code, Object messageObject) { + if (messageObject instanceof Throwable) { + super.initCause((Throwable)messageObject); + } this.code = code; this.messageObject = messageObject; + } public MMSException(int code, Object messageObject) { + if (messageObject instanceof Throwable) { + super.initCause((Throwable)messageObject); + } this.code = HttpStatus.resolve(code); this.messageObject = messageObject; } @@ -36,4 +43,12 @@ public void setMessageObject(Object messageObject) { this.messageObject = messageObject; } + @Override + public String getMessage() { + if (messageObject == null) { + return super.getMessage(); + } + return (messageObject instanceof Throwable) ? + ((Throwable)messageObject).getMessage() : messageObject.toString(); + } } \ No newline at end of file diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java index 784118ee3..72201fb06 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java @@ -166,7 +166,7 @@ public RefJson createBranch(String projectId, RefJson branch) { for (Node n: nodeRepository.findAllByDeleted(false)) { docIds.add(n.getDocId()); } - nodeIndex.addToRef(docIds); + try { nodeIndex.addToRef(docIds); } catch(Exception e) {} eventPublisher.forEach((pub) -> pub.publish( EventObject.create(projectId, branch.getId(), "branch_created", branch))); return branch; diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 5c6748fc1..482e052fa 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -214,7 +214,7 @@ public void commitChanges(NodeChangeInfo info) { if (json != null && !json.isEmpty()) { this.nodeIndex.indexAll(json.values()); } - this.nodeIndex.removeFromRef(info.getOldDocIds()); + try { this.nodeIndex.removeFromRef(info.getOldDocIds()); } catch(Exception e) {} this.commitIndex.index(cmjs); Optional existing = this.commitRepository.findByCommitId(cmjs.getId()); @@ -235,7 +235,7 @@ public void commitChanges(NodeChangeInfo info) { this.nodeRepository.saveAll(new ArrayList<>(nodes.values())); } catch (Exception e) { logger.error("commitChanges error: {}", e.getMessage()); - throw new InternalErrorException("Error committing changes."); + throw new InternalErrorException("Error committing changes: " + e.getMessage()); } eventPublisher.forEach((pub) -> pub.publish( EventObject.create(cmjs.getProjectId(), cmjs.getRefId(), "commit", cmjs))); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index 4d1b42c15..1ef83b76c 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -79,6 +79,7 @@ public void deleteById(String index, String docId) { try { client.delete(new DeleteRequest(index, docId), REQUEST_OPTIONS); } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } } @@ -91,6 +92,7 @@ public void deleteAll(String index, Collection jsons) { } client.bulk(bulkIndex, REQUEST_OPTIONS); } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } } @@ -102,6 +104,7 @@ public boolean existsById(String index, String docId) { getRequest.storedFields("_none_"); return client.exists(getRequest, REQUEST_OPTIONS); } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } } @@ -118,6 +121,7 @@ public Optional findById(String index, String docId) { return Optional.empty(); } } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } } @@ -145,6 +149,7 @@ public List findAllById(String index, Set docIds) { } return listOfResponses; } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } } @@ -176,6 +181,7 @@ public void index(String index, BaseJson json) { try { client.index(new IndexRequest(index).id(json.getDocId()).source(json), REQUEST_OPTIONS); } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } } @@ -196,6 +202,7 @@ public E update(String index, BaseJson json) { } } } catch (IOException e) { + logger.error(e.getMessage(), e); throw new InternalErrorException(e); } return response; @@ -211,6 +218,7 @@ protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { throw new InternalErrorException(failure); } } catch (IOException ioe) { + logger.error(ioe.getMessage(), ioe); throw new InternalErrorException(ioe); } }, getListener()); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java index 7592fbabe..61fdf0503 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java @@ -20,12 +20,12 @@ protected RefJson newInstance() { } @Override - public void indexAll(Collection jsons) throws MMSException { + public void indexAll(Collection jsons) { this.indexAll(getIndex(), jsons); } @Override - public void index(BaseJson json) throws MMSException { + public void index(BaseJson json) { this.index(getIndex(), json); } @@ -37,11 +37,11 @@ public List findAllById(Set docIds) { return this.findAllById(getIndex(), docIds); } - public void deleteById(String docId) throws MMSException { + public void deleteById(String docId) { this.deleteById(getIndex(), docId); } - public void deleteAll(Collection jsons) throws MMSException { + public void deleteAll(Collection jsons) { this.deleteAll(getIndex(), jsons); } @@ -50,7 +50,7 @@ public boolean existsById(String docId) { } @Override - public RefJson update(RefJson refJson) throws MMSException { + public RefJson update(RefJson refJson) { return this.update(getIndex(), refJson); } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index d22dbbae0..2f13e8a2d 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -15,6 +15,7 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.openmbee.mms.core.dao.CommitIndexDAO; +import org.openmbee.mms.core.exceptions.InternalErrorException; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.CommitJson; @@ -156,7 +157,8 @@ public List elementHistory(String nodeId, Set commitIds) { } return commits; } catch (IOException e) { - throw new RuntimeException(e); + logger.error(e.getMessage(), e); + throw new InternalErrorException(e); } } @@ -186,7 +188,8 @@ private List getDocs(String commitId) { } return rawCommits; } catch (IOException ioe) { - throw new RuntimeException(ioe); + logger.error(ioe.getMessage(), ioe); + throw new InternalErrorException(ioe); } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index ae196372f..8f1f31aad 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -23,7 +22,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.dao.NodeIndexDAO; -import org.openmbee.mms.core.exceptions.MMSException; +import org.openmbee.mms.core.exceptions.InternalErrorException; import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.ElementJson; @@ -36,11 +35,11 @@ protected ElementJson newInstance() { return new ElementJson(); } - public void indexAll(Collection jsons) throws MMSException { + public void indexAll(Collection jsons) { this.indexAll(getIndex(), jsons); } - public void index(BaseJson json) throws MMSException { + public void index(BaseJson json) { this.index(getIndex(), json); } @@ -52,11 +51,11 @@ public List findAllById(Set docIds) { return this.findAllById(getIndex(), docIds); } - public void deleteById(String docId) throws MMSException { + public void deleteById(String docId) { this.deleteById(getIndex(), docId); } - public void deleteAll(Collection jsons) throws MMSException { + public void deleteAll(Collection jsons) { this.deleteAll(getIndex(), jsons); } @@ -83,7 +82,8 @@ public Optional getByCommitId(String commitId, String nodeId) { ob.putAll(searchResponse.getHits().getAt(0).getSourceAsMap()); return Optional.of(ob); } catch (IOException e) { - throw new RuntimeException(e); + logger.error(e.getMessage(), e); + throw new InternalErrorException(e); } } @@ -145,7 +145,8 @@ public Optional getElementLessThanOrEqualTimestamp(String nodeId, } count += this.termLimit; } catch (IOException e) { - throw new RuntimeException(e); + logger.error(e.getMessage(), e); + throw new InternalErrorException(e); } } return Optional.empty(); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/ProjectElasticImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/ProjectElasticImpl.java index 402d2e1df..ccfc173f2 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/ProjectElasticImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/ProjectElasticImpl.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.dao.ProjectIndex; +import org.openmbee.mms.core.exceptions.InternalErrorException; import org.openmbee.mms.json.ProjectJson; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -57,7 +58,8 @@ public void create(ProjectJson project) { createIndex(metadataIndex); update(project); } catch (IOException e) { - throw new RuntimeException(e); + logger.error(e.getMessage(), e); + throw new InternalErrorException(e); } } @@ -83,7 +85,8 @@ public void delete(String projectId) { client.indices().delete(request2, RequestOptions.DEFAULT).isAcknowledged(); client.indices().delete(request3, RequestOptions.DEFAULT).isAcknowledged(); } catch (IOException e) { - throw new RuntimeException(e); + logger.error(e.getMessage(), e); + throw new InternalErrorException(e); } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/services/ElasticSearchService.java b/elastic/src/main/java/org/openmbee/mms/elastic/services/ElasticSearchService.java index e8f2ee158..3e9e1b1d3 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/services/ElasticSearchService.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/services/ElasticSearchService.java @@ -92,7 +92,8 @@ public ElementsSearchResponse recursiveSearch(String projectId, String refId, Ma Collection> filteredElementJson = filterIndexedElementsUsingDatabaseNodes(allNodes, elementJsonMap, deletedElements, showDeletedAsRejected); return prepareResponse(filteredElementJson, deletedElements, from, size); } catch (IOException e) { - throw new RuntimeException(e); + logger.error(e.getMessage(), e); + throw new InternalErrorException(e); } } diff --git a/example/elastic.postman_collection.json b/example/elastic.postman_collection.json index 992bfb2e1..62a68d9c7 100644 --- a/example/elastic.postman_collection.json +++ b/example/elastic.postman_collection.json @@ -157,7 +157,7 @@ "exec": [ "pm.test(\"response has element\", function () {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.elements.length).to.eql(1);", + " pm.expect(jsonData.elements.length).to.eql(2);", "});", "", "pm.environment.set(\"commitId1\", pm.response.json().elements[0]._commitId);" @@ -177,7 +177,7 @@ ], "body": { "mode": "raw", - "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": \"x\"\n\t\t}\n\t]\n}" + "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": \"x\"\n\t\t},\n {\n\t\t\t\"id\": \"z\",\n\t\t\t\"name\": \"z\"\n\t\t}\n\t]\n}" }, "url": { "raw": "{{host}}/projects/elastic/refs/master/elements", @@ -221,7 +221,7 @@ ], "body": { "mode": "raw", - "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": {}\n\t\t}\n\t]\n}" + "raw": "{\n\t\"elements\": [\n {\n\t\t\t\"id\": \"z\",\n\t\t\t\"name\": \"y\"\n\t\t},\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": {}\n\t\t}\n\t]\n}" }, "url": { "raw": "{{host}}/projects/elastic/refs/master/elements", @@ -284,6 +284,51 @@ }, "response": [] }, + { + "name": "get z", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "pm.test(\"response has element z\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.elements[0].name).to.eql('z');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements/z", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements", + "z" + ] + } + }, + "response": [] + }, { "name": "update element to elastic", "event": [ @@ -312,7 +357,7 @@ ], "body": { "mode": "raw", - "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": \"y\"\n\t\t}\n\t]\n}" + "raw": "{\n\t\"elements\": [\n\t\t{\n\t\t\t\"id\": \"x\",\n\t\t\t\"name\": \"y\"\n\t\t},\n {\n\t\t\t\"id\": \"z\",\n\t\t\t\"name\": \"y\"\n\t\t}\n\t]\n}" }, "url": { "raw": "{{host}}/projects/elastic/refs/master/elements", @@ -419,7 +464,7 @@ "response": [] }, { - "name": "check element commit history", + "name": "check element x commit history", "event": [ { "listen": "test", @@ -462,6 +507,51 @@ } }, "response": [] + }, + { + "name": "check element z commit history", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has 2 commit\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.commits.length).to.eql(2);", + "});", + "", + "pm.test(\"commit history is right\", function() {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId2\"))", + " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", + "});", + "", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/projects/elastic/refs/master/elements/z/commits", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "elastic", + "refs", + "master", + "elements", + "z", + "commits" + ] + } + }, + "response": [] } ], "auth": { From 8af289ab4412ef085b91790afef692a0551ff8b6 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Tue, 9 Feb 2021 18:14:31 -0800 Subject: [PATCH 11/11] clean import --- .../main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java index 61fdf0503..83ccb3e9b 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BranchElasticDAOImpl.java @@ -7,7 +7,6 @@ import java.util.UUID; import org.openmbee.mms.core.dao.BranchIndexDAO; -import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.RefJson; import org.springframework.stereotype.Component;