From 508f9b45d7d2869aba26e3a76f8c1a1a3e278f05 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 20 Jan 2021 11:33:49 -0800 Subject: [PATCH 01/19] Refactor to use commitId instead of docId and adding munging of associated commits --- .../openmbee/mms/core/dao/CommitIndexDAO.java | 2 + .../elements/ElementsController.java | 51 +++++- .../crud/services/DefaultBranchService.java | 2 +- .../crud/services/DefaultCommitService.java | 4 +- .../mms/crud/services/DefaultNodeService.java | 5 +- .../mms/crud/services/NodeGetHelper.java | 6 +- .../mms/crud/services/NodeOperation.java | 7 +- .../mms/data/domains/scoped/Commit.java | 10 +- .../mms/elastic/BaseElasticDAOImpl.java | 2 +- .../mms/elastic/CommitElasticDAOImpl.java | 165 ++++++++++++++---- .../resources/elastic_mappings/commit.json | 2 +- example/crud.postman_collection.json | 14 +- .../repositories/commit/CommitDAOImpl.java | 6 +- 13 files changed, 214 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java b/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java index 34926b267..efefc50e2 100644 --- a/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java +++ b/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java @@ -13,6 +13,8 @@ public interface CommitIndexDAO { void index(BaseJson json); + void index(BaseJson json, String commitId); + Optional findById(String docId); List findAllById(Set docIds); diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java index f711affd7..a07c63579 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java @@ -1,12 +1,21 @@ package org.openmbee.mms.crud.controllers.elements; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Objects; import org.openmbee.mms.core.objects.ElementsRequest; import org.openmbee.mms.core.objects.ElementsResponse; @@ -16,10 +25,10 @@ import org.openmbee.mms.core.services.NodeService; import org.openmbee.mms.core.pubsub.EmbeddedHookService; import org.openmbee.mms.crud.hooks.ElementUpdateHook; -import org.openmbee.mms.crud.services.DefaultCommitService; +import org.openmbee.mms.json.ElementJson; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.HttpStatus; +import org.springframework.http.HttpEntity; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -54,6 +63,8 @@ public void setCommitService(@Qualifier("defaultCommitService")CommitService com this.commitService = commitService; } + private static final JsonFactory jfactory = new JsonFactory(); + @GetMapping @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_READ', true)") @ApiResponse(responseCode = "200", content = { @@ -113,6 +124,42 @@ public ElementsResponse createOrUpdateElements( throw new BadRequestException(response.addMessage("Empty")); } + @PostMapping(value = "/stream", consumes = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_EDIT_CONTENT', false)") + public ResponseEntity createOrUpdateElementsStream( + @PathVariable String projectId, + @PathVariable String refId, + @RequestParam(required = false) Map params, + @Parameter(hidden = true) @RequestHeader(value = "Accept", defaultValue = "application/json") String accept, + Authentication auth, + HttpEntity requestEntity) { + + InputStream stream = new ByteArrayInputStream(Objects.requireNonNull(requestEntity.getBody())); + StreamingResponseBody response = outputStream -> { + ObjectMapper om = new ObjectMapper(); + try (JsonParser parser = jfactory.createParser(stream)) { + if(parser.nextToken() != JsonToken.START_OBJECT) { + throw new BadRequestException("Expected an object"); + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + logger.debug("Current Token: " + parser.getCurrentName()); + if (parser.nextToken() == JsonToken.START_ARRAY && "elements".equals(parser.getCurrentName())) { + logger.debug("Found Array: " + parser.getCurrentName()); + while (parser.nextToken() == JsonToken.START_OBJECT) { + ElementJson node = om.readValue(parser, ElementJson.class); + outputStream.write(node.get("id").toString().getBytes(StandardCharsets.UTF_8)); + } + } + } + } catch (IOException e) { + logger.debug("Error in stream handling: ", e); + } + }; + return ResponseEntity.ok() + .header("Content-Type", accept.equals("application/x-ndjson") ? accept : "application/json") + .body(response); + } + @PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_READ', true)") public ElementsResponse getElements( diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java index 3a7f4e813..6ab4b1d40 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java @@ -150,7 +150,7 @@ public RefJson createBranch(String projectId, RefJson branch) { Optional parentCommit = commitRepository.findLatestByRef(refOption.get()); parentCommit.ifPresent(parent -> { b.setParentCommit(parent.getId()); - branch.setParentCommitId(parent.getDocId()); //commit id is same as its docId + branch.setParentCommitId(parent.getCommitId()); //commit id is same as its docId }); } diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java index 863a9527a..9e96e1e08 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java @@ -77,7 +77,7 @@ public CommitsResponse getRefCommits(String projectId, String refId, Map refCommits = commitRepository.findByRefAndTimestampAndLimit(ref.get(), null, 0); Set commitIds = new HashSet<>(); for (Commit commit: refCommits) { - commitIds.add(commit.getDocId()); + commitIds.add(commit.getCommitId()); } res.getCommits().addAll(commitIndex.elementHistory(elementId, commitIds)); return res; diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index f660e6284..02205d612 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -224,12 +224,13 @@ protected void commitChanges(NodeChangeInfo info) { commit.setBranchId(cmjs.getRefId()); commit.setCommitType(CommitType.COMMIT); commit.setCreator(cmjs.getCreator()); - commit.setDocId(cmjs.getId()); + commit.setCommitId(cmjs.getCommitId()); commit.setTimestamp(now); commit.setComment(cmjs.getComment()); - + this.commitRepository.save(commit); this.commitIndex.index(cmjs); + this.nodeRepository.getTransactionManager().commit(status); } catch (Exception e) { logger.error("commitChanges error: ", e); diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java index c043ddd89..611238ce4 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java @@ -113,7 +113,7 @@ public NodeGetInfo processGetJson(List elements, Instant time, Node if (ref.isPresent()) { Optional c = commitRepository.findLatestByRef(ref.get()); if (c.isPresent()) { - return processGetJson(elements, c.get().getDocId(), service); + return processGetJson(elements, c.get().getCommitId(), service); } else { throw new BadRequestException("invalid time"); } @@ -148,7 +148,7 @@ public List processGetAll(Instant time, NodeService service) { if (ref.isPresent()) { Optional c = commitRepository.findByRefAndTimestamp(ref.get(), time); if (c.isPresent()) { - result.addAll(processGetAll(c.get().getDocId(), service)); + result.addAll(processGetAll(c.get().getCommitId(), service)); } else { throw new BadRequestException("invalid time"); } @@ -167,7 +167,7 @@ protected List getRefCommitIds(Instant time) { ref.ifPresent(current -> { List refCommits = commitRepository.findByRefAndTimestampAndLimit(current, time, 0); for (Commit c : refCommits) { - commitIds.add(c.getDocId()); + commitIds.add(c.getCommitId()); } }); return commitIds; diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index 284c5f124..8a7f2fd4f 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -66,6 +66,7 @@ public void setBranchRepository(BranchDAO branchRepository) { public void initCommitJson(CommitJson cmjs, Instant now) { cmjs.setId(UUID.randomUUID().toString()); + cmjs.setCommitId(UUID.randomUUID().toString()); cmjs.setDocId(cmjs.getId()); cmjs.setCreated(formatter.format(now)); cmjs.setAdded(new ArrayList<>()); @@ -130,7 +131,7 @@ public void processElementAdded(ElementJson e, Node n, NodeChangeInfo info) { cmjs.getAdded().add(newObj); n.setNodeId(e.getId()); - n.setInitialCommit(e.getDocId()); + n.setInitialCommit(e.getCommitId()); } public void processElementUpdated(ElementJson e, Node n, NodeChangeInfo info) { @@ -155,7 +156,7 @@ private void processElementAddedOrUpdated(ElementJson e, Node n, NodeChangeInfo e.setInRefIds(inRefIds); String docId = UUID.randomUUID().toString(); e.setDocId(docId); - e.setCommitId(cmjs.getId()); + e.setCommitId(cmjs.getCommitId()); if(!preserveTimestamps) { e.setModified(cmjs.getCreated()); @@ -163,7 +164,7 @@ private void processElementAddedOrUpdated(ElementJson e, Node n, NodeChangeInfo } n.setDocId(e.getDocId()); - n.setLastCommit(cmjs.getId()); + n.setLastCommit(cmjs.getCommitId()); n.setDeleted(false); n.setNodeType(0); diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java index 8d02bf217..875b5d2da 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java @@ -23,7 +23,7 @@ public class Commit implements Serializable { private Instant timestamp; @Column(unique = true) - private String docId; + private String commitId; private String branchId; private String creator; @@ -48,12 +48,12 @@ public void setTimestamp(Instant timestamp) { this.timestamp = timestamp; } - public String getDocId() { - return docId; + public String getCommitId() { + return commitId; } - public void setDocId(String docId) { - this.docId = docId; + public void setCommitId(String commitId) { + this.commitId = commitId; } public String getBranchId() { diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index c574a955e..8c59c2e1c 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -40,7 +40,7 @@ public abstract class BaseElasticDAOImpl> { - private final Logger logger = LoggerFactory.getLogger(getClass()); + final Logger logger = LoggerFactory.getLogger(getClass()); @Value("${elasticsearch.limit.result:10000}") protected int resultLimit; diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index cbc2add81..edb6b8344 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -1,12 +1,9 @@ package org.openmbee.mms.elastic; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; +import java.util.stream.Collectors; + import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -31,31 +28,39 @@ protected CommitJson newInstance() { } public void indexAll(Collection jsons) { - this.indexAll(getIndex(), jsons); + this.indexAll(getIndex(), associateCommits(jsons)); } public void index(BaseJson json) { + if (json.get(CommitJson.COMMITID).toString().isEmpty()) { + CommitJson commitJson = (CommitJson) json; + commitJson.put(CommitJson.COMMITID, UUID.randomUUID().toString()); + } this.index(getIndex(), json); } - public Optional findById(String docId) { - return this.findById(getIndex(), docId); + public void index(BaseJson json, String commitId) { + this.index(getIndex(), addCommitId(json, commitId)); } - public List findAllById(Set docIds) { - return this.findAllById(getIndex(), docIds); + public Optional findById(String commitId) { + return Optional.of(getFullCommit(commitId)); } - public void deleteById(String docId) { - this.deleteById(getIndex(), docId); + public List findAllById(Set commitIds) { + return getFullCommits(commitIds); + } + + public void deleteById(String commitId) { + this.deleteById(getIndex(), commitId); } public void deleteAll(Collection jsons) { this.deleteAll(getIndex(), jsons); } - public boolean existsById(String docId) { - return this.existsById(getIndex(), docId); + public boolean existsById(String commitId) { + return this.existsById(getIndex(), commitId); } /** @@ -75,14 +80,14 @@ private QueryBuilder getCommitHistoryQuery(String id, Set commitIds) { .should(addedQuery) .should(updatedQuery) .should(deletedQuery) - .filter(QueryBuilders.termsQuery(CommitJson.ID, commitIds)) + .filter(QueryBuilders.termsQuery(CommitJson.COMMITID, commitIds)) .minimumShouldMatch(1); return query; } /** * Returns the commit history of a element - *

Returns a list of commit metadata for the specificed id + *

Returns a list of commit metadata for the specified id * *

* @@ -93,26 +98,25 @@ private QueryBuilder getCommitHistoryQuery(String id, Set commitIds) { @Override public List elementHistory(String nodeId, Set commitIds) { try { - List commits = new ArrayList<>(); - SearchRequest searchRequest = new SearchRequest(getIndex()); QueryBuilder query = getCommitHistoryQuery(nodeId, commitIds); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.query(query); - sourceBuilder.size(this.resultLimit); // TODO handle paging requests - sourceBuilder.sort(new FieldSortBuilder(CommitJson.CREATED).order(SortOrder.DESC)); - searchRequest.source(sourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - SearchHits hits = searchResponse.getHits(); + SearchHits hits = getCommitResults(query); if (hits.getTotalHits().value == 0) { - return commits; + return new ArrayList<>(); } + LinkedHashMap> rawCommits = new LinkedHashMap<>(); for (SearchHit hit : hits.getHits()) { - Map source = hit.getSourceAsMap();// gets "_source" - CommitJson ob = newInstance(); - ob.putAll(source); - commits.add(ob); + CommitJson ob = new CommitJson(); + ob.putAll(hit.getSourceAsMap()); + if (!rawCommits.containsKey(ob.getCommitId())) { + rawCommits.put(ob.getCommitId(), new ArrayList<>()); + } + rawCommits.get(ob.getCommitId()).add(ob); // gets "_source" + } + ArrayList result = new ArrayList<>(); + for (String key : rawCommits.keySet()) { + result.add(mungCommits(rawCommits.get(key))); } - return commits; + return result; } catch (IOException e) { throw new RuntimeException(e); } @@ -127,4 +131,101 @@ protected String getIndex() { public CommitJson update(CommitJson commitJson) { return this.update(getIndex(), commitJson); } + + private Collection associateCommits(Collection jsons) { + String initialCommitId = jsons.stream().findFirst().orElseThrow().getCommitId(); + return jsons.stream().map(json -> addCommitId(json, initialCommitId)).collect(Collectors.toList()); + } + + private List getFullCommits(Collection commitIds) { + return commitIds.stream().map(this::getFullCommit).collect(Collectors.toList()); + } + + private CommitJson getFullCommit(String commitId) { + try { + QueryBuilder commitQuery = QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery(CommitJson.COMMITID, commitId)) + .should(QueryBuilders.termQuery(CommitJson.ID, commitId)) // Should it still be supported? + .minimumShouldMatch(1); + SearchHits hits = getCommitResults(commitQuery); + if (hits.getTotalHits().value == 0) { + return new CommitJson(); + } + List rawCommits = new ArrayList<>(); + for (SearchHit hit : hits.getHits()) { + CommitJson ob = new CommitJson(); + ob.putAll(hit.getSourceAsMap()); + rawCommits.add(ob); // gets "_source" + } + return mungCommits(rawCommits); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + private SearchHits getCommitResults(QueryBuilder query) throws IOException { + SearchRequest searchRequest = new SearchRequest(getIndex()); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(query); + sourceBuilder.size(this.resultLimit); // TODO handle paging requests + sourceBuilder.sort(new FieldSortBuilder(CommitJson.CREATED).order(SortOrder.DESC)); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse.getHits(); + } + + private static CommitJson mungCommits(List commits) { + return commits.stream().reduce(new CommitJson(), (partial, raw) -> { + if (partial.getAdded() == null) { + partial.setAdded(new ArrayList<>()); + } + if (partial.getUpdated() == null) { + partial.setUpdated(new ArrayList<>()); + } + if (partial.getDeleted() == null) { + partial.setDeleted(new ArrayList<>()); + } + if (partial.getSource() == null) { + partial.setSource(""); + } + if (partial.getComment() == null) { + partial.setComment(""); + } + if (partial.getId() == null) { + partial.setId(""); + } + if (partial.getCommitId() == null) { + partial.setCommitId(""); + } + + if (partial.getAdded().isEmpty() && raw.getAdded() != null) { + partial.getAdded().addAll(raw.getAdded()); + } + if (partial.getUpdated().isEmpty() && raw.getUpdated() != null) { + partial.getUpdated().addAll(raw.getUpdated()); + } + if (partial.getDeleted().isEmpty() && raw.getDeleted() != null) { + partial.getDeleted().addAll(raw.getDeleted()); + } + if (partial.getSource().isEmpty() && raw.getSource() != null) { + partial.setSource(raw.getSource()); + } + if (partial.getComment().isEmpty() && raw.getComment() != null) { + partial.setComment(raw.getComment()); + } + if (partial.getId().isEmpty() && raw.getCommitId() != null) { + partial.setId(raw.getCommitId()); + } + if (partial.getCommitId().isEmpty() && raw.getCommitId() != null) { + partial.setCommitId(raw.getCommitId()); + } + return partial; + }); + } + + @SuppressWarnings("unchecked") + // Should be the same type? + private static T addCommitId(T json, String associationId) { + return (T) json.put(CommitJson.COMMITID, associationId); + } } diff --git a/elastic/src/main/resources/elastic_mappings/commit.json b/elastic/src/main/resources/elastic_mappings/commit.json index 4d3a47dfb..194280c4f 100644 --- a/elastic/src/main/resources/elastic_mappings/commit.json +++ b/elastic/src/main/resources/elastic_mappings/commit.json @@ -59,7 +59,7 @@ "_projectId": { "type": "keyword" }, - "_docId": { + "_commitId": { "type": "keyword" }, "type": { diff --git a/example/crud.postman_collection.json b/example/crud.postman_collection.json index 826e78ede..be94c11e7 100644 --- a/example/crud.postman_collection.json +++ b/example/crud.postman_collection.json @@ -617,9 +617,9 @@ "", "pm.test(\"commit history is right\", function() {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId3\"))", - " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId2\"))", - " pm.expect(jsonData.commits[2].id).to.eql(pm.environment.get(\"commitId1\"))", + " pm.expect(jsonData.commits[0]._commitId).to.eql(pm.environment.get(\"commitId3\"))", + " pm.expect(jsonData.commits[1]._commitId).to.eql(pm.environment.get(\"commitId2\"))", + " pm.expect(jsonData.commits[2]._commitId).to.eql(pm.environment.get(\"commitId1\"))", "})", "", "" @@ -664,8 +664,8 @@ "", "pm.test(\"commit history is right\", function() {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId3\"))", - " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", + " pm.expect(jsonData.commits[0]._commitId).to.eql(pm.environment.get(\"commitId3\"))", + " pm.expect(jsonData.commits[1]._commitId).to.eql(pm.environment.get(\"commitId1\"))", "})", "", "" @@ -710,8 +710,8 @@ "", "pm.test(\"commit history is right\", function() {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId4\"))", - " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", + " pm.expect(jsonData.commits[0]._commitId).to.eql(pm.environment.get(\"commitId4\"))", + " pm.expect(jsonData.commits[1]._commitId).to.eql(pm.environment.get(\"commitId1\"))", "})", "" ], diff --git a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/commit/CommitDAOImpl.java b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/commit/CommitDAOImpl.java index 4c94490e2..eb4e55fdc 100644 --- a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/commit/CommitDAOImpl.java +++ b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/commit/CommitDAOImpl.java @@ -33,7 +33,7 @@ public void setBranchRepository(BranchDAO branchRepository) { } public Commit save(Commit commit) { - String sql = "INSERT INTO commits (commitType, creator, docid, branchId, timestamp, comment) VALUES (?, ?, ?, ?, ?, ?)"; + String sql = "INSERT INTO commits (commitType, creator, commitid, branchId, timestamp, comment) VALUES (?, ?, ?, ?, ?, ?)"; KeyHolder keyHolder = new GeneratedKeyHolder(); getConn().update(new PreparedStatementCreator() { @@ -42,7 +42,7 @@ public PreparedStatement createPreparedStatement(Connection connection) PreparedStatement ps = connection.prepareStatement(sql, new String[]{"id"}); ps.setInt(1, commit.getCommitType().getId()); ps.setString(2, commit.getCreator()); - ps.setString(3, commit.getDocId()); + ps.setString(3, commit.getCommitId()); ps.setString(4, commit.getBranchId()); ps.setTimestamp(5, Timestamp.from(commit.getTimestamp())); ps.setString(6, commit.getComment()); @@ -69,7 +69,7 @@ public Optional findById(long id) { } public Optional findByCommitId(String commitId) { - String sql = "SELECT * FROM commits WHERE docid = ?"; + String sql = "SELECT * FROM commits WHERE commitid = ?"; List l = getConn() .query(sql, new Object[]{commitId}, new CommitRowMapper()); From 60ebac58ea40422ae5810979b47a56edbf8e42cf Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 21 Jan 2021 17:38:42 -0800 Subject: [PATCH 02/19] Add commitId support for existing commits --- .../elements/ElementsController.java | 24 +++++++++++---- .../mms/crud/services/DefaultNodeService.java | 29 +++++++++++++------ .../mms/elastic/BaseElasticDAOImpl.java | 3 +- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java index a07c63579..44afbe2ac 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java @@ -13,9 +13,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.Objects; +import java.util.*; import org.openmbee.mms.core.objects.ElementsRequest; import org.openmbee.mms.core.objects.ElementsResponse; @@ -28,6 +26,7 @@ import org.openmbee.mms.json.ElementJson; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -53,6 +52,9 @@ public class ElementsController extends BaseController { private EmbeddedHookService embeddedHookService; private CommitService commitService; + @Value("${mms.stream.batch.size:5000}") + private int streamLimit; + @Autowired public void setEmbeddedHookService(EmbeddedHookService embeddedHookService) { this.embeddedHookService = embeddedHookService; @@ -134,6 +136,11 @@ public ResponseEntity createOrUpdateElementsStream( Authentication auth, HttpEntity requestEntity) { + String commitId = UUID.randomUUID().toString(); // Generate a commitId from the start + params.put("commitId", commitId); + ElementsRequest req = new ElementsRequest(); + List elements = new ArrayList<>(); + InputStream stream = new ByteArrayInputStream(Objects.requireNonNull(requestEntity.getBody())); StreamingResponseBody response = outputStream -> { ObjectMapper om = new ObjectMapper(); @@ -145,12 +152,19 @@ public ResponseEntity createOrUpdateElementsStream( logger.debug("Current Token: " + parser.getCurrentName()); if (parser.nextToken() == JsonToken.START_ARRAY && "elements".equals(parser.getCurrentName())) { logger.debug("Found Array: " + parser.getCurrentName()); - while (parser.nextToken() == JsonToken.START_OBJECT) { + while (parser.nextToken() != JsonToken.END_OBJECT) { ElementJson node = om.readValue(parser, ElementJson.class); - outputStream.write(node.get("id").toString().getBytes(StandardCharsets.UTF_8)); + elements.add(node); + //outputStream.write(node.getType().getBytes(StandardCharsets.UTF_8)); + //outputStream.write(node.get("id").toString().getBytes(StandardCharsets.UTF_8)); } } } + req.setElements(elements); + if (!req.getElements().isEmpty()) { + NodeService nodeService = getNodeService(projectId); + nodeService.createOrUpdate(projectId, refId, req, params, auth.getName()); + } } catch (IOException e) { logger.debug("Error in stream handling: ", e); } diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 02205d612..dd3ef58cd 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -194,6 +194,11 @@ public ElementsResponse createOrUpdate(String projectId, String refId, ElementsR .processPostJson(req.getElements(), overwriteJson, createCommit(user, refId, projectId, req), this); + String commitId = params.get("commitId"); + if (commitId != null && !commitId.isEmpty()) { + info.getCommitJson().setCommitId(commitId); + } + commitChanges(info); ElementsResponse response = new ElementsResponse(); @@ -220,15 +225,21 @@ protected void commitChanges(NodeChangeInfo info) { } this.nodeIndex.removeFromRef(info.getOldDocIds()); - Commit commit = new Commit(); - commit.setBranchId(cmjs.getRefId()); - commit.setCommitType(CommitType.COMMIT); - commit.setCreator(cmjs.getCreator()); - commit.setCommitId(cmjs.getCommitId()); - commit.setTimestamp(now); - commit.setComment(cmjs.getComment()); - - this.commitRepository.save(commit); + Optional existing = this.commitRepository.findByCommitId(cmjs.getCommitId()); + existing.ifPresentOrElse( + current -> { + this.logger.debug(String.format("Commit object %s already exists. Skipping record creation.", current.getCommitId())); + }, + () -> { + Commit commit = new Commit(); + commit.setBranchId(cmjs.getRefId()); + commit.setCommitType(CommitType.COMMIT); + commit.setCreator(cmjs.getCreator()); + commit.setCommitId(cmjs.getCommitId()); + commit.setTimestamp(now); + commit.setComment(cmjs.getComment()); + this.commitRepository.save(commit); + }); this.commitIndex.index(cmjs); this.nodeRepository.getTransactionManager().commit(status); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index 8c59c2e1c..0d328248d 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -181,8 +181,7 @@ public void indexAll(String index, Collection jsons) { public void index(String index, BaseJson json) { try { - client.index(new IndexRequest(index).id(json.getDocId()).source(json), - REQUEST_OPTIONS); + client.index(new IndexRequest(index).id(json.getDocId()).source(json), REQUEST_OPTIONS); } catch (IOException e) { throw new RuntimeException(e); } From d2491c7af3a94361d723044a3b1081bdd197c02e Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Fri, 22 Jan 2021 17:36:13 -0800 Subject: [PATCH 03/19] use bulk processor for elastic script update on branch create --- .../mms/crud/services/DefaultBranchService.java | 1 + .../openmbee/mms/elastic/BaseElasticDAOImpl.java | 2 +- .../openmbee/mms/elastic/NodeElasticDAOImpl.java | 15 +++++++++------ .../services/TwcRevisionMmsCommitMapService.java | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java index 6ab4b1d40..784118ee3 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java @@ -172,6 +172,7 @@ public RefJson createBranch(String projectId, RefJson branch) { return branch; } catch (Exception e) { logger.error("Couldn't create branch: {}", branch.getId(), e); + //TODO should clean up any created tables/rows? throw new InternalErrorException(e); } } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index 0d328248d..f75de49bb 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -208,7 +208,7 @@ public E update(String index, BaseJson json) { return response; } - private BulkProcessor getBulkProcessor(RestHighLevelClient client) { + protected BulkProcessor getBulkProcessor(RestHighLevelClient client) { return getBulkProcessor(client, null); } diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java index 5088a8ec6..2804f20b8 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/NodeElasticDAOImpl.java @@ -7,7 +7,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.elasticsearch.action.bulk.BulkRequest; +import java.util.concurrent.TimeUnit; +import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -101,7 +102,7 @@ private void bulkUpdateRefWithScript(Set docIds, String script) { if (docIds.isEmpty()) { return; } - BulkRequest bulk = new BulkRequest(); + BulkProcessor bulkProcessor = getBulkProcessor(client); Map parameters = Collections.singletonMap("refId", ContextHolder.getContext().getBranchId()); for (String docId : docIds) { @@ -109,12 +110,14 @@ private void bulkUpdateRefWithScript(Set docIds, String script) { Script inline = new Script(ScriptType.INLINE, "painless", script, parameters); request.script(inline); - bulk.add(request); + bulkProcessor.add(request); } try { - client.bulk(bulk, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException(e); + if (!bulkProcessor.awaitClose(1200L, TimeUnit.SECONDS)) { + logger.error("Timed out in bulk processing"); + } + } catch (InterruptedException e) { + logger.error("Index all interrupted: ", e); } } diff --git a/twc/src/main/java/org/openmbee/mms/twc/services/TwcRevisionMmsCommitMapService.java b/twc/src/main/java/org/openmbee/mms/twc/services/TwcRevisionMmsCommitMapService.java index 1d689a0ed..6361a60bd 100644 --- a/twc/src/main/java/org/openmbee/mms/twc/services/TwcRevisionMmsCommitMapService.java +++ b/twc/src/main/java/org/openmbee/mms/twc/services/TwcRevisionMmsCommitMapService.java @@ -75,7 +75,7 @@ public List getTwcRevisionList(String projectId, String refId, Boole List refCommits = commitRepository.findByRefAndTimestampAndLimit(ref.get(), null, 0); Set commitIds = new HashSet<>(); refCommits.stream().forEach(commit -> { - commitIds.add(commit.getDocId()); + commitIds.add(commit.getCommitId()); }); List commitJsonList = commitIndex.findAllById(commitIds); if (null != commitJsonList && commitJsonList.size() > 0) { From fb1970f83022274825c0465661ed9252ba462c51 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Mon, 25 Jan 2021 10:28:22 -0800 Subject: [PATCH 04/19] Break up commits by size internally and move commit id setting --- .../openmbee/mms/core/dao/CommitIndexDAO.java | 2 - .../mms/crud/services/DefaultNodeService.java | 17 +-- .../mms/crud/services/NodeOperation.java | 4 +- .../mms/elastic/CommitElasticDAOImpl.java | 112 ++++++++++++++---- json/json.gradle | 2 + 5 files changed, 100 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java b/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java index efefc50e2..34926b267 100644 --- a/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java +++ b/core/src/main/java/org/openmbee/mms/core/dao/CommitIndexDAO.java @@ -13,8 +13,6 @@ public interface CommitIndexDAO { void index(BaseJson json); - void index(BaseJson json, String commitId); - Optional findById(String docId); List findAllById(Set docIds); diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index dd3ef58cd..b95e7e127 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -189,15 +189,11 @@ public ElementsResponse createOrUpdate(String projectId, String refId, ElementsR ContextHolder.setContext(projectId, refId); boolean overwriteJson = Boolean.parseBoolean(params.get("overwrite")); nodePostHelper.setPreserveTimestamps(Boolean.parseBoolean(params.get("preserveTimestamps"))); + String commitId = params.get("commitId"); NodeChangeInfo info = nodePostHelper .processPostJson(req.getElements(), overwriteJson, - createCommit(user, refId, projectId, req), this); - - String commitId = params.get("commitId"); - if (commitId != null && !commitId.isEmpty()) { - info.getCommitJson().setCommitId(commitId); - } + createCommit(user, refId, projectId, req, commitId), this); commitChanges(info); @@ -294,7 +290,7 @@ public ElementsResponse delete(String projectId, String refId, ElementsRequest r ContextHolder.setContext(projectId, refId); NodeChangeInfo info = nodeDeleteHelper - .processDeleteJson(req.getElements(), createCommit(user, refId, projectId, req), + .processDeleteJson(req.getElements(), createCommit(user, refId, projectId, req, null), this); ElementsResponse response = new ElementsResponse(); @@ -306,13 +302,18 @@ public ElementsResponse delete(String projectId, String refId, ElementsRequest r } private CommitJson createCommit(String creator, String refId, String projectId, - ElementsRequest req) { + ElementsRequest req, String commitId) { CommitJson cmjs = new CommitJson(); cmjs.setCreator(creator); cmjs.setComment(req.getComment()); cmjs.setSource(req.getSource()); cmjs.setRefId(refId); cmjs.setProjectId(projectId); + + if (commitId != null && !commitId.isEmpty()) { + cmjs.setCommitId(commitId); + } + return cmjs; } diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index 8a7f2fd4f..38c47f632 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -66,7 +66,9 @@ public void setBranchRepository(BranchDAO branchRepository) { public void initCommitJson(CommitJson cmjs, Instant now) { cmjs.setId(UUID.randomUUID().toString()); - cmjs.setCommitId(UUID.randomUUID().toString()); + if (cmjs.getCommitId() == null || cmjs.getCommitId().isEmpty()) { + cmjs.setCommitId(UUID.randomUUID().toString()); + } cmjs.setDocId(cmjs.getId()); cmjs.setCreated(formatter.format(now)); cmjs.setAdded(new ArrayList<>()); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index edb6b8344..bc8f32763 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -18,29 +18,74 @@ import org.openmbee.mms.elastic.utils.Index; import org.openmbee.mms.json.BaseJson; import org.openmbee.mms.json.CommitJson; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component public class CommitElasticDAOImpl extends BaseElasticDAOImpl implements CommitIndexDAO { + @Value("${elasticsearch.limit.index}") + int indexLimit; + protected CommitJson newInstance() { return new CommitJson(); } public void indexAll(Collection jsons) { - this.indexAll(getIndex(), associateCommits(jsons)); + this.indexAll(getIndex(), jsons); } public void index(BaseJson json) { - if (json.get(CommitJson.COMMITID).toString().isEmpty()) { - CommitJson commitJson = (CommitJson) json; - commitJson.put(CommitJson.COMMITID, UUID.randomUUID().toString()); - } - this.index(getIndex(), json); + index((CommitJson) json); } - public void index(BaseJson json, String commitId) { - this.index(getIndex(), addCommitId(json, commitId)); + public void index(CommitJson json) { + if (json.getCommitId().isEmpty()) { + json.setCommitId(UUID.randomUUID().toString()); + } + + int commitCount = getCommitSize(json); + List broken = new ArrayList<>(); + if (commitCount > indexLimit) { + List> allActions = new ArrayList<>(); + allActions.addAll(json.getAdded().stream().peek(toAdd -> toAdd.put("action", "added")).collect(Collectors.toList())); + allActions.addAll(json.getUpdated().stream().peek(toUpdate -> toUpdate.put("action", "updated")).collect(Collectors.toList())); + allActions.addAll(json.getDeleted().stream().peek(toDelete -> toDelete.put("action", "deleted")).collect(Collectors.toList())); + + while (!allActions.isEmpty()) { + CommitJson currentCommitCopy = new CommitJson(); + currentCommitCopy.setCommitId(json.getCommitId()); + currentCommitCopy.setSource(json.getSource()); + currentCommitCopy.setCreated(json.getCreated()); + currentCommitCopy.setCreator(json.getCreator()); + currentCommitCopy.setModified(json.getModified()); + currentCommitCopy.setModifier(json.getModifier()); + currentCommitCopy.setAdded(new ArrayList<>()); + currentCommitCopy.setUpdated(new ArrayList<>()); + currentCommitCopy.setDeleted(new ArrayList<>()); + + while (getCommitSize(currentCommitCopy) < indexLimit && !allActions.isEmpty()) { + Map action = allActions.remove(0); + switch (action.getOrDefault("action", "none").toString()) { + case "added": + currentCommitCopy.getAdded().add(action); + break; + case "updated": + currentCommitCopy.getUpdated().add(action); + break; + case "deleted": + currentCommitCopy.getDeleted().add(action); + break; + } + } + broken.add(currentCommitCopy); + + } + this.indexAll(broken); + + } else { + this.index(getIndex(), json); + } } public Optional findById(String commitId) { @@ -52,7 +97,10 @@ public List findAllById(Set commitIds) { } public void deleteById(String commitId) { - this.deleteById(getIndex(), commitId); + List docs = getDocs(commitId); + docs.forEach(commit -> { + this.deleteById(getIndex(), commit.getDocId()); + }); } public void deleteAll(Collection jsons) { @@ -60,7 +108,12 @@ public void deleteAll(Collection jsons) { } public boolean existsById(String commitId) { - return this.existsById(getIndex(), commitId); + List docs = getDocs(commitId); + List result = new ArrayList<>(); + docs.forEach(commit -> { + result.add(this.existsById(getIndex(), commit.getDocId())); + }); + return !result.stream().allMatch(b -> b); } /** @@ -132,16 +185,7 @@ public CommitJson update(CommitJson commitJson) { return this.update(getIndex(), commitJson); } - private Collection associateCommits(Collection jsons) { - String initialCommitId = jsons.stream().findFirst().orElseThrow().getCommitId(); - return jsons.stream().map(json -> addCommitId(json, initialCommitId)).collect(Collectors.toList()); - } - - private List getFullCommits(Collection commitIds) { - return commitIds.stream().map(this::getFullCommit).collect(Collectors.toList()); - } - - private CommitJson getFullCommit(String commitId) { + private List getDocs(String commitId) { try { QueryBuilder commitQuery = QueryBuilders.boolQuery() .should(QueryBuilders.termQuery(CommitJson.COMMITID, commitId)) @@ -149,7 +193,7 @@ private CommitJson getFullCommit(String commitId) { .minimumShouldMatch(1); SearchHits hits = getCommitResults(commitQuery); if (hits.getTotalHits().value == 0) { - return new CommitJson(); + return new ArrayList<>(); } List rawCommits = new ArrayList<>(); for (SearchHit hit : hits.getHits()) { @@ -157,12 +201,20 @@ private CommitJson getFullCommit(String commitId) { ob.putAll(hit.getSourceAsMap()); rawCommits.add(ob); // gets "_source" } - return mungCommits(rawCommits); + return rawCommits; } catch (IOException ioe) { throw new RuntimeException(ioe); } } + private List getFullCommits(Collection commitIds) { + return commitIds.stream().map(this::getFullCommit).collect(Collectors.toList()); + } + + private CommitJson getFullCommit(String commitId) { + return mungCommits(getDocs(commitId)); + } + private SearchHits getCommitResults(QueryBuilder query) throws IOException { SearchRequest searchRequest = new SearchRequest(getIndex()); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); @@ -223,9 +275,17 @@ private static CommitJson mungCommits(List commits) { }); } - @SuppressWarnings("unchecked") - // Should be the same type? - private static T addCommitId(T json, String associationId) { - return (T) json.put(CommitJson.COMMITID, associationId); + private static int getCommitSize(CommitJson commitJson) { + int commitCount = 0; + if (commitJson.getAdded() != null) { + commitCount = commitCount + commitJson.getAdded().size(); + } + if (commitJson.getUpdated() != null) { + commitCount = commitCount + commitJson.getUpdated().size(); + } + if (commitJson.getDeleted() != null) { + commitCount = commitCount + commitJson.getDeleted().size(); + } + return commitCount; } } diff --git a/json/json.gradle b/json/json.gradle index 88de0b9ad..0daf19acc 100644 --- a/json/json.gradle +++ b/json/json.gradle @@ -1,4 +1,6 @@ dependencies { + api commonDependencies.'jackson-databind' + implementation commonDependencies.'jackson-annotations' implementation commonDependencies.'swagger-annotations' testImplementation commonDependencies.'spring-boot-starter-test' From 989188e41176a3ba9067a2377b0e640076bb469b Mon Sep 17 00:00:00 2001 From: Jason Han Date: Mon, 25 Jan 2021 11:35:30 -0800 Subject: [PATCH 05/19] Assign id to commit id, if not set --- .../main/java/org/openmbee/mms/crud/services/NodeOperation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index 38c47f632..299281d86 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -67,7 +67,7 @@ public void setBranchRepository(BranchDAO branchRepository) { public void initCommitJson(CommitJson cmjs, Instant now) { cmjs.setId(UUID.randomUUID().toString()); if (cmjs.getCommitId() == null || cmjs.getCommitId().isEmpty()) { - cmjs.setCommitId(UUID.randomUUID().toString()); + cmjs.setCommitId(cmjs.getId()); } cmjs.setDocId(cmjs.getId()); cmjs.setCreated(formatter.format(now)); From aa9fa37013dea27ce04c1a051be2639d085a8aea Mon Sep 17 00:00:00 2001 From: Jason Han Date: Mon, 25 Jan 2021 11:47:45 -0800 Subject: [PATCH 06/19] Increase delay for commit indexing --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 534002bdd..6be3cec43 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,13 +31,13 @@ jobs: command: | docker create -v /etc/newman --name mms_test_configs alpine:3.4 /bin/true docker cp example/. mms_test_configs:/etc/newman - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 300 - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 300 - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 300 - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 300 - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 300 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 500 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 500 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 500 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 500 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 500 docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run search.postman_collection.json -e test-env.json --delay-request 1000 - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 300 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 500 - persist_to_workspace: root: /home/circleci/ From c7bdbb85237eefe1feeda630908add9d5cf42a7c Mon Sep 17 00:00:00 2001 From: Jason Han Date: Mon, 25 Jan 2021 12:45:40 -0800 Subject: [PATCH 07/19] Increase delay for cameo tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6be3cec43..c8f9dc3ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: docker create -v /etc/newman --name mms_test_configs alpine:3.4 /bin/true docker cp example/. mms_test_configs:/etc/newman docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 500 - docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 500 + docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 1000 docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 500 docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 500 docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 500 From bff1da405be7ce82f06fc0122d2583f6fef5721e Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 26 Jan 2021 08:23:37 -0800 Subject: [PATCH 08/19] Refactor to use commit object's id as commit id --- .../mms/crud/services/DefaultNodeService.java | 4 +- .../mms/crud/services/NodeOperation.java | 5 +- .../mms/elastic/CommitElasticDAOImpl.java | 35 ++++-------- example/crud.postman_collection.json | 57 +++---------------- .../org/openmbee/mms/json/CommitJson.java | 18 ++++++ 5 files changed, 39 insertions(+), 80 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index b95e7e127..2b12afd43 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -228,10 +228,10 @@ protected void commitChanges(NodeChangeInfo info) { }, () -> { Commit commit = new Commit(); + commit.setCommitId(cmjs.getId()); commit.setBranchId(cmjs.getRefId()); commit.setCommitType(CommitType.COMMIT); commit.setCreator(cmjs.getCreator()); - commit.setCommitId(cmjs.getCommitId()); commit.setTimestamp(now); commit.setComment(cmjs.getComment()); this.commitRepository.save(commit); @@ -311,7 +311,7 @@ private CommitJson createCommit(String creator, String refId, String projectId, cmjs.setProjectId(projectId); if (commitId != null && !commitId.isEmpty()) { - cmjs.setCommitId(commitId); + cmjs.setId(commitId); } return cmjs; diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index 299281d86..cc3ac96db 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -66,9 +66,6 @@ public void setBranchRepository(BranchDAO branchRepository) { public void initCommitJson(CommitJson cmjs, Instant now) { cmjs.setId(UUID.randomUUID().toString()); - if (cmjs.getCommitId() == null || cmjs.getCommitId().isEmpty()) { - cmjs.setCommitId(cmjs.getId()); - } cmjs.setDocId(cmjs.getId()); cmjs.setCreated(formatter.format(now)); cmjs.setAdded(new ArrayList<>()); @@ -158,7 +155,7 @@ private void processElementAddedOrUpdated(ElementJson e, Node n, NodeChangeInfo e.setInRefIds(inRefIds); String docId = UUID.randomUUID().toString(); e.setDocId(docId); - e.setCommitId(cmjs.getCommitId()); + e.setCommitId(cmjs.getId()); if(!preserveTimestamps) { e.setModified(cmjs.getCreated()); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index bc8f32763..c2e0329bd 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -40,10 +40,6 @@ public void index(BaseJson json) { } public void index(CommitJson json) { - if (json.getCommitId().isEmpty()) { - json.setCommitId(UUID.randomUUID().toString()); - } - int commitCount = getCommitSize(json); List broken = new ArrayList<>(); if (commitCount > indexLimit) { @@ -53,17 +49,7 @@ public void index(CommitJson json) { allActions.addAll(json.getDeleted().stream().peek(toDelete -> toDelete.put("action", "deleted")).collect(Collectors.toList())); while (!allActions.isEmpty()) { - CommitJson currentCommitCopy = new CommitJson(); - currentCommitCopy.setCommitId(json.getCommitId()); - currentCommitCopy.setSource(json.getSource()); - currentCommitCopy.setCreated(json.getCreated()); - currentCommitCopy.setCreator(json.getCreator()); - currentCommitCopy.setModified(json.getModified()); - currentCommitCopy.setModifier(json.getModifier()); - currentCommitCopy.setAdded(new ArrayList<>()); - currentCommitCopy.setUpdated(new ArrayList<>()); - currentCommitCopy.setDeleted(new ArrayList<>()); - + CommitJson currentCommitCopy = CommitJson.copy(json); while (getCommitSize(currentCommitCopy) < indexLimit && !allActions.isEmpty()) { Map action = allActions.remove(0); switch (action.getOrDefault("action", "none").toString()) { @@ -133,7 +119,7 @@ private QueryBuilder getCommitHistoryQuery(String id, Set commitIds) { .should(addedQuery) .should(updatedQuery) .should(deletedQuery) - .filter(QueryBuilders.termsQuery(CommitJson.COMMITID, commitIds)) + .filter(QueryBuilders.termsQuery(CommitJson.ID, commitIds)) .minimumShouldMatch(1); return query; } @@ -160,10 +146,10 @@ public List elementHistory(String nodeId, Set commitIds) { for (SearchHit hit : hits.getHits()) { CommitJson ob = new CommitJson(); ob.putAll(hit.getSourceAsMap()); - if (!rawCommits.containsKey(ob.getCommitId())) { - rawCommits.put(ob.getCommitId(), new ArrayList<>()); + if (!rawCommits.containsKey(ob.getId())) { + rawCommits.put(ob.getId(), new ArrayList<>()); } - rawCommits.get(ob.getCommitId()).add(ob); // gets "_source" + rawCommits.get(ob.getId()).add(ob); // gets "_source" } ArrayList result = new ArrayList<>(); for (String key : rawCommits.keySet()) { @@ -188,8 +174,7 @@ public CommitJson update(CommitJson commitJson) { private List getDocs(String commitId) { try { QueryBuilder commitQuery = QueryBuilders.boolQuery() - .should(QueryBuilders.termQuery(CommitJson.COMMITID, commitId)) - .should(QueryBuilders.termQuery(CommitJson.ID, commitId)) // Should it still be supported? + .should(QueryBuilders.termQuery(CommitJson.ID, commitId)) .minimumShouldMatch(1); SearchHits hits = getCommitResults(commitQuery); if (hits.getTotalHits().value == 0) { @@ -246,8 +231,8 @@ private static CommitJson mungCommits(List commits) { if (partial.getId() == null) { partial.setId(""); } - if (partial.getCommitId() == null) { - partial.setCommitId(""); + if (partial.getId() == null) { + partial.setId(""); } if (partial.getAdded().isEmpty() && raw.getAdded() != null) { @@ -268,8 +253,8 @@ private static CommitJson mungCommits(List commits) { if (partial.getId().isEmpty() && raw.getCommitId() != null) { partial.setId(raw.getCommitId()); } - if (partial.getCommitId().isEmpty() && raw.getCommitId() != null) { - partial.setCommitId(raw.getCommitId()); + if (partial.getId().isEmpty() && raw.getId() != null) { + partial.setId(raw.getId()); } return partial; }); diff --git a/example/crud.postman_collection.json b/example/crud.postman_collection.json index be94c11e7..5782b5322 100644 --- a/example/crud.postman_collection.json +++ b/example/crud.postman_collection.json @@ -617,9 +617,9 @@ "", "pm.test(\"commit history is right\", function() {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.commits[0]._commitId).to.eql(pm.environment.get(\"commitId3\"))", - " pm.expect(jsonData.commits[1]._commitId).to.eql(pm.environment.get(\"commitId2\"))", - " pm.expect(jsonData.commits[2]._commitId).to.eql(pm.environment.get(\"commitId1\"))", + " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId3\"))", + " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId2\"))", + " pm.expect(jsonData.commits[2].id).to.eql(pm.environment.get(\"commitId1\"))", "})", "", "" @@ -664,8 +664,8 @@ "", "pm.test(\"commit history is right\", function() {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.commits[0]._commitId).to.eql(pm.environment.get(\"commitId3\"))", - " pm.expect(jsonData.commits[1]._commitId).to.eql(pm.environment.get(\"commitId1\"))", + " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId3\"))", + " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", "})", "", "" @@ -710,8 +710,8 @@ "", "pm.test(\"commit history is right\", function() {", " var jsonData = pm.response.json();", - " pm.expect(jsonData.commits[0]._commitId).to.eql(pm.environment.get(\"commitId4\"))", - " pm.expect(jsonData.commits[1]._commitId).to.eql(pm.environment.get(\"commitId1\"))", + " pm.expect(jsonData.commits[0].id).to.eql(pm.environment.get(\"commitId4\"))", + " pm.expect(jsonData.commits[1].id).to.eql(pm.environment.get(\"commitId1\"))", "})", "" ], @@ -2467,54 +2467,13 @@ }, "response": [] }, - { - "name": "recreate project cc under org a", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"response has project cc\", function () {", - " var jsonData = pm.response.json();", - " pm.expect(jsonData.projects[0].id).to.eql('cc');", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "type": "text", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n\t\"projects\": [\n\t\t{\n\t\t\t\"id\": \"cc\", \n\t\t\t\"name\": \"cc\",\n\t\t\t\"orgId\": \"a\",\n\t\t\t\"schema\": \"default\"\n\t\t}\n\t]\n}" - }, - "url": { - "raw": "{{host}}/projects", - "host": [ - "{{host}}" - ], - "path": [ - "projects" - ] - } - }, - "response": [] - }, { "name": "create webhook on project aa", "event": [ { "listen": "test", "script": { - "id": "2ba7a873-1bdf-4e86-8329-7069064232a9", + "id": "2ba7a873-1bdf-4e86-8329-7069064232a9", "exec": [ "pm.test(\"webhook created with returned id\", function () {", " var jsonData = pm.response.json();", diff --git a/json/src/main/java/org/openmbee/mms/json/CommitJson.java b/json/src/main/java/org/openmbee/mms/json/CommitJson.java index c41e50c2f..20665728b 100644 --- a/json/src/main/java/org/openmbee/mms/json/CommitJson.java +++ b/json/src/main/java/org/openmbee/mms/json/CommitJson.java @@ -4,6 +4,8 @@ import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema.AccessMode; + +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -17,6 +19,22 @@ public class CommitJson extends BaseJson { public static final String UPDATED = "updated"; public static final String SOURCE = "source"; + public static CommitJson copy(CommitJson original) { + CommitJson copy = new CommitJson(); + copy.setId(original.getId()); + copy.setSource(original.getSource()); + copy.setCreated(original.getCreated()); + copy.setCreator(original.getCreator()); + copy.setModified(original.getModified()); + copy.setModifier(original.getModifier()); + copy.setRefId(original.getRefId()); + copy.setProjectId(original.getProjectId()); + copy.setAdded(new ArrayList<>()); + copy.setUpdated(new ArrayList<>()); + copy.setDeleted(new ArrayList<>()); + return copy; + } + @Schema(accessMode = AccessMode.READ_ONLY) public String getComment() { return (String) this.get(COMMENT); From 9d507b9d8a07f6dee8d4e251fde46bc8481ba0ec Mon Sep 17 00:00:00 2001 From: Jason Han Date: Tue, 26 Jan 2021 14:08:46 -0800 Subject: [PATCH 09/19] Remove action attribute fix more commit id references --- build.gradle | 2 +- .../elements/ElementsController.java | 3 ++- .../mms/crud/services/DefaultNodeService.java | 2 +- .../mms/crud/services/NodeOperation.java | 10 +++++++--- .../mms/elastic/CommitElasticDAOImpl.java | 17 ++++++++--------- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/build.gradle b/build.gradle index 7c02a7ca2..82c35ae8e 100644 --- a/build.gradle +++ b/build.gradle @@ -6,8 +6,8 @@ buildscript { } repositories { mavenCentral() - maven { url 'https://repo.spring.io/plugins-release' } gradlePluginPortal() + maven { url 'https://repo.spring.io/plugins-release' } } } diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java index 44afbe2ac..23bb377a2 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java @@ -125,9 +125,10 @@ public ElementsResponse createOrUpdateElements( } throw new BadRequestException(response.addMessage("Empty")); } - + /* @PostMapping(value = "/stream", consumes = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_EDIT_CONTENT', false)") + */ public ResponseEntity createOrUpdateElementsStream( @PathVariable String projectId, @PathVariable String refId, diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index 2b12afd43..19ca028a6 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -221,7 +221,7 @@ protected void commitChanges(NodeChangeInfo info) { } this.nodeIndex.removeFromRef(info.getOldDocIds()); - Optional existing = this.commitRepository.findByCommitId(cmjs.getCommitId()); + Optional existing = this.commitRepository.findByCommitId(cmjs.getId()); existing.ifPresentOrElse( current -> { this.logger.debug(String.format("Commit object %s already exists. Skipping record creation.", current.getCommitId())); diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index cc3ac96db..92e7b77d4 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -65,8 +65,12 @@ public void setBranchRepository(BranchDAO branchRepository) { } public void initCommitJson(CommitJson cmjs, Instant now) { - cmjs.setId(UUID.randomUUID().toString()); - cmjs.setDocId(cmjs.getId()); + if (cmjs.getId() == null || cmjs.getId().isEmpty()) { + cmjs.setId(UUID.randomUUID().toString()); + } + if (cmjs.getDocId() == null || cmjs.getDocId().isEmpty()) { + cmjs.setDocId(UUID.randomUUID().toString()); + } cmjs.setCreated(formatter.format(now)); cmjs.setAdded(new ArrayList<>()); cmjs.setDeleted(new ArrayList<>()); @@ -163,7 +167,7 @@ private void processElementAddedOrUpdated(ElementJson e, Node n, NodeChangeInfo } n.setDocId(e.getDocId()); - n.setLastCommit(cmjs.getCommitId()); + n.setLastCommit(cmjs.getId()); n.setDeleted(false); n.setNodeType(0); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index c2e0329bd..f9f0bf8d2 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -52,7 +52,9 @@ public void index(CommitJson json) { CommitJson currentCommitCopy = CommitJson.copy(json); while (getCommitSize(currentCommitCopy) < indexLimit && !allActions.isEmpty()) { Map action = allActions.remove(0); - switch (action.getOrDefault("action", "none").toString()) { + String compare = action.getOrDefault("action", "none").toString(); + action.remove("action"); + switch (compare) { case "added": currentCommitCopy.getAdded().add(action); break; @@ -174,8 +176,7 @@ public CommitJson update(CommitJson commitJson) { private List getDocs(String commitId) { try { QueryBuilder commitQuery = QueryBuilders.boolQuery() - .should(QueryBuilders.termQuery(CommitJson.ID, commitId)) - .minimumShouldMatch(1); + .filter(QueryBuilders.termQuery(CommitJson.ID, commitId)); SearchHits hits = getCommitResults(commitQuery); if (hits.getTotalHits().value == 0) { return new ArrayList<>(); @@ -231,19 +232,17 @@ private static CommitJson mungCommits(List commits) { if (partial.getId() == null) { partial.setId(""); } - if (partial.getId() == null) { - partial.setId(""); - } - if (partial.getAdded().isEmpty() && raw.getAdded() != null) { + if (raw.getAdded() != null) { partial.getAdded().addAll(raw.getAdded()); } - if (partial.getUpdated().isEmpty() && raw.getUpdated() != null) { + if (raw.getUpdated() != null) { partial.getUpdated().addAll(raw.getUpdated()); } - if (partial.getDeleted().isEmpty() && raw.getDeleted() != null) { + if (raw.getDeleted() != null) { partial.getDeleted().addAll(raw.getDeleted()); } + if (partial.getSource().isEmpty() && raw.getSource() != null) { partial.setSource(raw.getSource()); } From 35538b592687de2f658d6f9552cb51caf5859da7 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 27 Jan 2021 09:41:27 -0800 Subject: [PATCH 10/19] Optimize element history and change fields to text instead of varchar --- .../mms/data/domains/scoped/Branch.java | 4 ++- .../mms/data/domains/scoped/Commit.java | 4 ++- .../mms/data/domains/scoped/Node.java | 2 +- .../mms/elastic/CommitElasticDAOImpl.java | 29 +++++++++---------- .../resources/application.properties.example | 1 + .../org/openmbee/mms/json/CommitJson.java | 1 + 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java index cf1013d5d..559833fa1 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java @@ -19,11 +19,13 @@ public class Branch { private String description; - @Column(unique = true) + @Column(unique = true, columnDefinition = "TEXT") private String branchId; private String docId; + @Column(columnDefinition = "TEXT") private String branchName; + @Column(columnDefinition = "TEXT") private String parentRefId; private Long parentCommit; diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java index 875b5d2da..37c2c15f6 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java @@ -24,9 +24,11 @@ public class Commit implements Serializable { @Column(unique = true) private String commitId; + @Column(columnDefinition = "TEXT") private String branchId; private String creator; + @Column(columnDefinition = "TEXT") private String comment; @Column(columnDefinition = "smallint") @@ -81,7 +83,7 @@ public void setCommitType(CommitType commitType) { } public String getComment() { - return comment; + return comment == null ? "" : comment; } public void setComment(String comment) { diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java index 66cbc33d3..3f7e2b567 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java @@ -16,7 +16,7 @@ public class Node { @Column(name = "id", updatable = false, nullable = false) Long id; - @Column(unique = true) + @Column(unique = true, columnDefinition = "TEXT") private String nodeId; private String docId; private String lastCommit; diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index f9f0bf8d2..8d0a07654 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -24,8 +24,8 @@ @Component public class CommitElasticDAOImpl extends BaseElasticDAOImpl implements CommitIndexDAO { - @Value("${elasticsearch.limit.index}") - int indexLimit; + @Value("${elasticsearch.limit.commit:10000}") + int commitLimit; protected CommitJson newInstance() { return new CommitJson(); @@ -42,7 +42,7 @@ public void index(BaseJson json) { public void index(CommitJson json) { int commitCount = getCommitSize(json); List broken = new ArrayList<>(); - if (commitCount > indexLimit) { + if (commitCount > commitLimit) { List> allActions = new ArrayList<>(); allActions.addAll(json.getAdded().stream().peek(toAdd -> toAdd.put("action", "added")).collect(Collectors.toList())); allActions.addAll(json.getUpdated().stream().peek(toUpdate -> toUpdate.put("action", "updated")).collect(Collectors.toList())); @@ -50,7 +50,7 @@ public void index(CommitJson json) { while (!allActions.isEmpty()) { CommitJson currentCommitCopy = CommitJson.copy(json); - while (getCommitSize(currentCommitCopy) < indexLimit && !allActions.isEmpty()) { + while (getCommitSize(currentCommitCopy) < commitLimit && !allActions.isEmpty()) { Map action = allActions.remove(0); String compare = action.getOrDefault("action", "none").toString(); action.remove("action"); @@ -139,25 +139,22 @@ private QueryBuilder getCommitHistoryQuery(String id, Set commitIds) { @Override public List elementHistory(String nodeId, Set commitIds) { try { + List commits = new ArrayList<>(); QueryBuilder query = getCommitHistoryQuery(nodeId, commitIds); SearchHits hits = getCommitResults(query); if (hits.getTotalHits().value == 0) { return new ArrayList<>(); } - LinkedHashMap> rawCommits = new LinkedHashMap<>(); for (SearchHit hit : hits.getHits()) { - CommitJson ob = new CommitJson(); - ob.putAll(hit.getSourceAsMap()); - if (!rawCommits.containsKey(ob.getId())) { - rawCommits.put(ob.getId(), new ArrayList<>()); - } - rawCommits.get(ob.getId()).add(ob); // gets "_source" - } - ArrayList result = new ArrayList<>(); - for (String key : rawCommits.keySet()) { - result.add(mungCommits(rawCommits.get(key))); + Map source = hit.getSourceAsMap();// gets "_source" + CommitJson ob = newInstance(); + ob.putAll(source); + ob.remove(CommitJson.ADDED); + ob.remove(CommitJson.UPDATED); + ob.remove(CommitJson.DELETED); + commits.add(ob); } - return result; + return commits; } catch (IOException e) { throw new RuntimeException(e); } diff --git a/example/src/main/resources/application.properties.example b/example/src/main/resources/application.properties.example index 001b74fce..367f032d3 100644 --- a/example/src/main/resources/application.properties.example +++ b/example/src/main/resources/application.properties.example @@ -53,6 +53,7 @@ elasticsearch.limit.term=1000 elasticsearch.limit.scrollTimeout=1000 elasticsearch.limit.get=100000 elasticsearch.limit.index=5000 +elasticsearch.limit.commit=100000 #Configuration for TWC #port is for REST interface diff --git a/json/src/main/java/org/openmbee/mms/json/CommitJson.java b/json/src/main/java/org/openmbee/mms/json/CommitJson.java index 20665728b..cdef92518 100644 --- a/json/src/main/java/org/openmbee/mms/json/CommitJson.java +++ b/json/src/main/java/org/openmbee/mms/json/CommitJson.java @@ -27,6 +27,7 @@ public static CommitJson copy(CommitJson original) { copy.setCreator(original.getCreator()); copy.setModified(original.getModified()); copy.setModifier(original.getModifier()); + copy.setComment(original.getComment()); copy.setRefId(original.getRefId()); copy.setProjectId(original.getProjectId()); copy.setAdded(new ArrayList<>()); From 9aad0690cdc7f2f1f25d5479c6b93ea74f90456c Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 27 Jan 2021 10:04:16 -0800 Subject: [PATCH 11/19] Revert commitId elastic mapping --- elastic/src/main/resources/elastic_mappings/commit.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/src/main/resources/elastic_mappings/commit.json b/elastic/src/main/resources/elastic_mappings/commit.json index 194280c4f..4d3a47dfb 100644 --- a/elastic/src/main/resources/elastic_mappings/commit.json +++ b/elastic/src/main/resources/elastic_mappings/commit.json @@ -59,7 +59,7 @@ "_projectId": { "type": "keyword" }, - "_commitId": { + "_docId": { "type": "keyword" }, "type": { From 85dae16ea609af9051848e90a6ccac1830079e78 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 27 Jan 2021 10:28:06 -0800 Subject: [PATCH 12/19] Change length of fields and revert deleted test --- .../mms/data/domains/scoped/Branch.java | 6 +-- .../mms/data/domains/scoped/Commit.java | 4 +- .../mms/data/domains/scoped/Node.java | 2 +- example/crud.postman_collection.json | 41 +++++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java index 559833fa1..b1f609f83 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java @@ -19,13 +19,13 @@ public class Branch { private String description; - @Column(unique = true, columnDefinition = "TEXT") + @Column(unique = true, length = 512) private String branchId; private String docId; - @Column(columnDefinition = "TEXT") + @Column(length = 512) private String branchName; - @Column(columnDefinition = "TEXT") + @Column(length = 512) private String parentRefId; private Long parentCommit; diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java index 37c2c15f6..f21eec2e2 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Commit.java @@ -24,11 +24,11 @@ public class Commit implements Serializable { @Column(unique = true) private String commitId; - @Column(columnDefinition = "TEXT") + @Column(length = 512) private String branchId; private String creator; - @Column(columnDefinition = "TEXT") + @Column(length = 512) private String comment; @Column(columnDefinition = "smallint") diff --git a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java index 3f7e2b567..1359c9d9f 100644 --- a/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java +++ b/data/src/main/java/org/openmbee/mms/data/domains/scoped/Node.java @@ -16,7 +16,7 @@ public class Node { @Column(name = "id", updatable = false, nullable = false) Long id; - @Column(unique = true, columnDefinition = "TEXT") + @Column(unique = true, length = 512) private String nodeId; private String docId; private String lastCommit; diff --git a/example/crud.postman_collection.json b/example/crud.postman_collection.json index 5782b5322..0c2da825a 100644 --- a/example/crud.postman_collection.json +++ b/example/crud.postman_collection.json @@ -2467,6 +2467,47 @@ }, "response": [] }, + { + "name": "recreate project cc under org a", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has project cc\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.projects[0].id).to.eql('cc');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"projects\": [\n\t\t{\n\t\t\t\"id\": \"cc\", \n\t\t\t\"name\": \"cc\",\n\t\t\t\"orgId\": \"a\",\n\t\t\t\"schema\": \"default\"\n\t\t}\n\t]\n}" + }, + "url": { + "raw": "{{host}}/projects", + "host": [ + "{{host}}" + ], + "path": [ + "projects" + ] + } + }, + "response": [] + }, { "name": "create webhook on project aa", "event": [ From 5e60981c329eedb7cea26474a29f433d41639fb9 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 27 Jan 2021 10:54:06 -0800 Subject: [PATCH 13/19] Add parameter support for projects endpoint --- .../projects/ProjectsController.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java index ebf197ee0..f970828ce 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java @@ -1,10 +1,7 @@ package org.openmbee.mms.crud.controllers.projects; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; + import org.openmbee.mms.core.config.ContextHolder; import org.openmbee.mms.core.config.Privileges; import org.openmbee.mms.core.config.ProjectSchemas; @@ -35,6 +32,12 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + @RestController @RequestMapping("/projects") @Tag(name = "Projects") @@ -55,16 +58,22 @@ public ProjectsController(ProjectDAO projectRepository, ProjectIndex projectInde } @GetMapping - public ProjectsResponse getAllProjects(Authentication auth) { + public ProjectsResponse getAllProjects(Authentication auth, + @RequestParam(required = false) Map params) { ProjectsResponse response = new ProjectsResponse(); List allProjects = projectRepository.findAll(); + String orgId = params.getOrDefault("orgId", null); for (Project proj : allProjects) { if (mss.hasProjectPrivilege(auth, proj.getProjectId(), Privileges.PROJECT_READ.name(), true)) { ContextHolder.setContext(proj.getProjectId()); if(proj.getDocId() != null && !proj.isDeleted()) { Optional projectJsonOption = projectIndex.findById(proj.getDocId()); - projectJsonOption.ifPresentOrElse(json -> response.getProjects().add(json), ()-> { + projectJsonOption.ifPresentOrElse(json -> { + if (orgId == null || json.getOrgId().equals(orgId)) { + response.getProjects().add(json); + } + }, ()-> { logger.error("Project json not found for id: {}", proj.getProjectId()); }); } From 68cc229cdef47bac352a688abc4ca5f940a84b34 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 27 Jan 2021 11:31:34 -0800 Subject: [PATCH 14/19] Implement get project by org id in project repository and define orgId as parameter --- .../java/org/openmbee/mms/core/dao/ProjectDAO.java | 2 ++ .../controllers/projects/ProjectsController.java | 12 +++--------- .../mms/rdb/repositories/ProjectDAOImpl.java | 5 +++++ .../mms/rdb/repositories/ProjectRepository.java | 5 +++++ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/openmbee/mms/core/dao/ProjectDAO.java b/core/src/main/java/org/openmbee/mms/core/dao/ProjectDAO.java index c90f0072b..244f543c9 100644 --- a/core/src/main/java/org/openmbee/mms/core/dao/ProjectDAO.java +++ b/core/src/main/java/org/openmbee/mms/core/dao/ProjectDAO.java @@ -15,4 +15,6 @@ public interface ProjectDAO { void delete(Project p); List findAll(); + + List findAllByOrgId(String id); } diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java index f970828ce..8dc62b6a9 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java @@ -59,21 +59,15 @@ public ProjectsController(ProjectDAO projectRepository, ProjectIndex projectInde @GetMapping public ProjectsResponse getAllProjects(Authentication auth, - @RequestParam(required = false) Map params) { - + @RequestParam(required = false) String orgId) { ProjectsResponse response = new ProjectsResponse(); - List allProjects = projectRepository.findAll(); - String orgId = params.getOrDefault("orgId", null); + List allProjects = orgId != null ? projectRepository.findAllByOrgId(orgId) : projectRepository.findAll(); for (Project proj : allProjects) { if (mss.hasProjectPrivilege(auth, proj.getProjectId(), Privileges.PROJECT_READ.name(), true)) { ContextHolder.setContext(proj.getProjectId()); if(proj.getDocId() != null && !proj.isDeleted()) { Optional projectJsonOption = projectIndex.findById(proj.getDocId()); - projectJsonOption.ifPresentOrElse(json -> { - if (orgId == null || json.getOrgId().equals(orgId)) { - response.getProjects().add(json); - } - }, ()-> { + projectJsonOption.ifPresentOrElse(json -> response.getProjects().add(json), ()-> { logger.error("Project json not found for id: {}", proj.getProjectId()); }); } diff --git a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectDAOImpl.java b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectDAOImpl.java index f7421d412..427cd2f81 100644 --- a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectDAOImpl.java +++ b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectDAOImpl.java @@ -41,6 +41,11 @@ public Optional findByProjectName(String name) { return projectRepository.findByProjectName(name); } + @Override + public List findAllByOrgId(String id) { + return projectRepository.findAllByOrganizationOrganizationId(id); + } + @Override public Project save(Project proj) { if (proj.getId() == null) { diff --git a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectRepository.java b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectRepository.java index a726afeeb..be7b12cfc 100644 --- a/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectRepository.java +++ b/rdb/src/main/java/org/openmbee/mms/rdb/repositories/ProjectRepository.java @@ -1,6 +1,9 @@ package org.openmbee.mms.rdb.repositories; +import java.util.List; import java.util.Optional; + +import org.openmbee.mms.data.domains.global.Organization; import org.openmbee.mms.data.domains.global.Project; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @@ -12,4 +15,6 @@ public interface ProjectRepository extends JpaRepository { Optional findByProjectName(String name); + List findAllByOrganizationOrganizationId(String id); + } From 21261798ce7fe7e164faab1e7a3a8166408536bc Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 27 Jan 2021 14:49:24 -0800 Subject: [PATCH 15/19] add test for get project by org, add creator,created,refid,projectid to munge commits --- .../mms/crud/services/NodeOperation.java | 1 + .../mms/elastic/CommitElasticDAOImpl.java | 28 ++++++++-- example/crud.postman_collection.json | 52 +++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index 92e7b77d4..fe41caa25 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -67,6 +67,7 @@ public void setBranchRepository(BranchDAO branchRepository) { public void initCommitJson(CommitJson cmjs, Instant now) { if (cmjs.getId() == null || cmjs.getId().isEmpty()) { cmjs.setId(UUID.randomUUID().toString()); + cmjs.setDocId(cmjs.getId()); } if (cmjs.getDocId() == null || cmjs.getDocId().isEmpty()) { cmjs.setDocId(UUID.randomUUID().toString()); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index 8d0a07654..369d253dd 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -50,6 +50,7 @@ public void index(CommitJson json) { while (!allActions.isEmpty()) { CommitJson currentCommitCopy = CommitJson.copy(json); + currentCommitCopy.setDocId(UUID.randomUUID().toString()); while (getCommitSize(currentCommitCopy) < commitLimit && !allActions.isEmpty()) { Map action = allActions.remove(0); String compare = action.getOrDefault("action", "none").toString(); @@ -229,6 +230,18 @@ private static CommitJson mungCommits(List commits) { if (partial.getId() == null) { partial.setId(""); } + if (partial.getCreated() == null) { + partial.setCreated(""); + } + if (partial.getCreator() == null) { + partial.setCreator(""); + } + if (partial.getRefId() == null) { + partial.setRefId(""); + } + if (partial.getProjectId() == null) { + partial.setProjectId(""); + } if (raw.getAdded() != null) { partial.getAdded().addAll(raw.getAdded()); @@ -246,12 +259,21 @@ private static CommitJson mungCommits(List commits) { if (partial.getComment().isEmpty() && raw.getComment() != null) { partial.setComment(raw.getComment()); } - if (partial.getId().isEmpty() && raw.getCommitId() != null) { - partial.setId(raw.getCommitId()); - } if (partial.getId().isEmpty() && raw.getId() != null) { partial.setId(raw.getId()); } + if (partial.getCreated().isEmpty() && raw.getCreated() != null) { + partial.setCreated(raw.getCreated()); + } + if (partial.getCreator().isEmpty() && raw.getCreator() != null) { + partial.setCreator(raw.getCreator()); + } + if (partial.getRefId().isEmpty() && raw.getRefId() != null) { + partial.setRefId(raw.getRefId()); + } + if (partial.getProjectId().isEmpty() && raw.getProjectId() != null) { + partial.setProjectId(raw.getProjectId()); + } return partial; }); } diff --git a/example/crud.postman_collection.json b/example/crud.postman_collection.json index 0c2da825a..3bdc34cbd 100644 --- a/example/crud.postman_collection.json +++ b/example/crud.postman_collection.json @@ -2872,6 +2872,58 @@ }, "response": [] }, + { + "name": "get org c projects", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response has 1 project random\", function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData.projects.length).to.eql(1);", + " pm.expect(jsonData.projects[0].name).to.eql('random');", + "});" + ], + "type": "text/javascript" + } + } + ], + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "url": { + "raw": "{{host}}/projects?orgId={{orgIdC}}", + "host": [ + "{{host}}" + ], + "path": [ + "projects" + ], + "query": [ + { + "key": "orgId", + "value": "{{orgIdC}}" + } + ] + } + }, + "response": [] + }, { "name": "add element random to project random without id", "event": [ From 01c4ecdc07aecfc97ad3a90fa4793d3fb0921597 Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Wed, 27 Jan 2021 16:34:58 -0800 Subject: [PATCH 16/19] add test for getting nonexisting commit, fix optional return --- .../mms/elastic/CommitElasticDAOImpl.java | 13 ++++-- example/crud.postman_collection.json | 41 ++++++++++++++++++- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index 369d253dd..23a819afb 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -78,7 +78,7 @@ public void index(CommitJson json) { } public Optional findById(String commitId) { - return Optional.of(getFullCommit(commitId)); + return getFullCommit(commitId); } public List findAllById(Set commitIds) { @@ -192,11 +192,16 @@ private List getDocs(String commitId) { } private List getFullCommits(Collection commitIds) { - return commitIds.stream().map(this::getFullCommit).collect(Collectors.toList()); + return commitIds.stream().map(this::getFullCommit).filter(Optional::isPresent) + .map(Optional::get).collect(Collectors.toList()); } - private CommitJson getFullCommit(String commitId) { - return mungCommits(getDocs(commitId)); + private Optional getFullCommit(String commitId) { + List commits = getDocs(commitId); + if (commits.isEmpty()) { + return Optional.empty(); + } + return Optional.of(mungCommits(commits)); } private SearchHits getCommitResults(QueryBuilder query) throws IOException { diff --git a/example/crud.postman_collection.json b/example/crud.postman_collection.json index 3bdc34cbd..a3f223e61 100644 --- a/example/crud.postman_collection.json +++ b/example/crud.postman_collection.json @@ -3438,7 +3438,46 @@ } }, "response": [] - } + }, + { + "name": "get nonexistent commit", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 404\", function () {", + " pm.response.to.have.status(404);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "type": "text", + "value": "application/json" + } + ], + "url": { + "raw": "{{host}}/projects/aa/commits/missing", + "host": [ + "{{host}}" + ], + "path": [ + "projects", + "aa", + "commits", + "missing" + ] + } + }, + "response": [] + } ], "auth": { "type": "bearer", From 8772ad3b9ec23f3098b13f11fb628d914cd1a778 Mon Sep 17 00:00:00 2001 From: Jason Han Date: Wed, 27 Jan 2021 17:18:06 -0800 Subject: [PATCH 17/19] Clean up copy method --- .../mms/elastic/CommitElasticDAOImpl.java | 81 +++---------------- .../org/openmbee/mms/json/CommitJson.java | 39 +++++---- 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index 369d253dd..7f7fcafcd 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -49,9 +49,12 @@ public void index(CommitJson json) { allActions.addAll(json.getDeleted().stream().peek(toDelete -> toDelete.put("action", "deleted")).collect(Collectors.toList())); while (!allActions.isEmpty()) { - CommitJson currentCommitCopy = CommitJson.copy(json); + CommitJson currentCommitCopy = CommitJson.copy(new CommitJson(), json); + currentCommitCopy.setAdded(new ArrayList<>()); + currentCommitCopy.setUpdated(new ArrayList<>()); + currentCommitCopy.setDeleted(new ArrayList<>()); currentCommitCopy.setDocId(UUID.randomUUID().toString()); - while (getCommitSize(currentCommitCopy) < commitLimit && !allActions.isEmpty()) { + do { Map action = allActions.remove(0); String compare = action.getOrDefault("action", "none").toString(); action.remove("action"); @@ -66,7 +69,7 @@ public void index(CommitJson json) { currentCommitCopy.getDeleted().add(action); break; } - } + } while(getCommitSize(currentCommitCopy) < commitLimit && !allActions.isEmpty()); broken.add(currentCommitCopy); } @@ -211,82 +214,18 @@ private SearchHits getCommitResults(QueryBuilder query) throws IOException { } private static CommitJson mungCommits(List commits) { - return commits.stream().reduce(new CommitJson(), (partial, raw) -> { - if (partial.getAdded() == null) { - partial.setAdded(new ArrayList<>()); - } - if (partial.getUpdated() == null) { - partial.setUpdated(new ArrayList<>()); - } - if (partial.getDeleted() == null) { - partial.setDeleted(new ArrayList<>()); - } - if (partial.getSource() == null) { - partial.setSource(""); - } - if (partial.getComment() == null) { - partial.setComment(""); - } - if (partial.getId() == null) { - partial.setId(""); - } - if (partial.getCreated() == null) { - partial.setCreated(""); - } - if (partial.getCreator() == null) { - partial.setCreator(""); - } - if (partial.getRefId() == null) { - partial.setRefId(""); - } - if (partial.getProjectId() == null) { - partial.setProjectId(""); - } - - if (raw.getAdded() != null) { - partial.getAdded().addAll(raw.getAdded()); - } - if (raw.getUpdated() != null) { - partial.getUpdated().addAll(raw.getUpdated()); - } - if (raw.getDeleted() != null) { - partial.getDeleted().addAll(raw.getDeleted()); - } - - if (partial.getSource().isEmpty() && raw.getSource() != null) { - partial.setSource(raw.getSource()); - } - if (partial.getComment().isEmpty() && raw.getComment() != null) { - partial.setComment(raw.getComment()); - } - if (partial.getId().isEmpty() && raw.getId() != null) { - partial.setId(raw.getId()); - } - if (partial.getCreated().isEmpty() && raw.getCreated() != null) { - partial.setCreated(raw.getCreated()); - } - if (partial.getCreator().isEmpty() && raw.getCreator() != null) { - partial.setCreator(raw.getCreator()); - } - if (partial.getRefId().isEmpty() && raw.getRefId() != null) { - partial.setRefId(raw.getRefId()); - } - if (partial.getProjectId().isEmpty() && raw.getProjectId() != null) { - partial.setProjectId(raw.getProjectId()); - } - return partial; - }); + return commits.stream().reduce(new CommitJson(), CommitJson::copy); } private static int getCommitSize(CommitJson commitJson) { int commitCount = 0; - if (commitJson.getAdded() != null) { + if (commitJson.getAdded() != null && !commitJson.getAdded().isEmpty()) { commitCount = commitCount + commitJson.getAdded().size(); } - if (commitJson.getUpdated() != null) { + if (commitJson.getUpdated() != null && !commitJson.getUpdated().isEmpty()) { commitCount = commitCount + commitJson.getUpdated().size(); } - if (commitJson.getDeleted() != null) { + if (commitJson.getDeleted() != null && !commitJson.getDeleted().isEmpty()) { commitCount = commitCount + commitJson.getDeleted().size(); } return commitCount; diff --git a/json/src/main/java/org/openmbee/mms/json/CommitJson.java b/json/src/main/java/org/openmbee/mms/json/CommitJson.java index cdef92518..2a8ac6160 100644 --- a/json/src/main/java/org/openmbee/mms/json/CommitJson.java +++ b/json/src/main/java/org/openmbee/mms/json/CommitJson.java @@ -19,20 +19,31 @@ public class CommitJson extends BaseJson { public static final String UPDATED = "updated"; public static final String SOURCE = "source"; - public static CommitJson copy(CommitJson original) { - CommitJson copy = new CommitJson(); - copy.setId(original.getId()); - copy.setSource(original.getSource()); - copy.setCreated(original.getCreated()); - copy.setCreator(original.getCreator()); - copy.setModified(original.getModified()); - copy.setModifier(original.getModifier()); - copy.setComment(original.getComment()); - copy.setRefId(original.getRefId()); - copy.setProjectId(original.getProjectId()); - copy.setAdded(new ArrayList<>()); - copy.setUpdated(new ArrayList<>()); - copy.setDeleted(new ArrayList<>()); + public static CommitJson copy(CommitJson copy, CommitJson original) { + if (copy.getAdded() == null) { + copy.setAdded(new ArrayList<>()); + } + if (copy.getUpdated() == null) { + copy.setUpdated(new ArrayList<>()); + } + if (copy.getDeleted() == null) { + copy.setDeleted(new ArrayList<>()); + } + + if (original.getAdded() != null) { + copy.getAdded().addAll(original.getAdded()); + original.remove(CommitJson.ADDED); + } + if (original.getUpdated() != null) { + copy.getUpdated().addAll(original.getUpdated()); + original.remove(CommitJson.UPDATED); + } + if (original.getDeleted() != null) { + copy.getDeleted().addAll(original.getDeleted()); + original.remove(CommitJson.DELETED); + } + + copy.putAll(original); return copy; } From 54b27f2e0ac787c63721fbb1cbea4bd519adcb1b Mon Sep 17 00:00:00 2001 From: "Lam, Doris T (319E)" Date: Thu, 28 Jan 2021 09:53:13 -0800 Subject: [PATCH 18/19] fill in ref and project id for ref commits --- .../org/openmbee/mms/crud/services/DefaultCommitService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java index 9e96e1e08..6479e146b 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java @@ -79,6 +79,8 @@ public CommitsResponse getRefCommits(String projectId, String refId, Map { From 1015cd125960c2e76cbb64175dc69b8c4661a48a Mon Sep 17 00:00:00 2001 From: Jason Han Date: Thu, 28 Jan 2021 11:26:43 -0800 Subject: [PATCH 19/19] Simplify existsById method for commits --- .../java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java index 92419c534..d22dbbae0 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/CommitElasticDAOImpl.java @@ -101,11 +101,7 @@ public void deleteAll(Collection jsons) { public boolean existsById(String commitId) { List docs = getDocs(commitId); - List result = new ArrayList<>(); - docs.forEach(commit -> { - result.add(this.existsById(getIndex(), commit.getDocId())); - }); - return !result.stream().allMatch(b -> b); + return !docs.isEmpty(); } /**