Feature/commit refactor

.circleci/config.yml

@@ -31,13 +31,13 @@ jobs:
           command: |
             docker create -v /etc/newman --name mms_test_configs alpine:3.4 /bin/true
             docker cp example/. mms_test_configs:/etc/newman
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 300
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 300
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 300
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 300
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 300
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run crud.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run cameo.postman_collection.json -e test-env.json --delay-request 1000
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run jupyter.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run localauth.postman_collection.json -e test-env.json --delay-request 500
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run permissions.postman_collection.json -e test-env.json --delay-request 500
             docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run search.postman_collection.json -e test-env.json --delay-request 1000
-            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 300
+            docker run --volumes-from mms_test_configs --network container:mms -t postman/newman run artifacts.postman_collection.json -e test-env.json --delay-request 500
 
       - persist_to_workspace:
             root: /home/circleci/

build.gradle

@@ -6,8 +6,8 @@ buildscript {
     }
     repositories {
         mavenCentral()
-        maven { url 'https://repo.spring.io/plugins-release' }
         gradlePluginPortal()
+        maven { url 'https://repo.spring.io/plugins-release' }
     }
 }
 

core/src/main/java/org/openmbee/mms/core/dao/ProjectDAO.java

@@ -15,4 +15,6 @@ public interface ProjectDAO {
     void delete(Project p);
 
     List<Project> findAll();
+
+    List<Project> findAllByOrgId(String id);
 }

crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java

@@ -1,12 +1,19 @@
 package org.openmbee.mms.crud.controllers.elements;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
-import java.util.Map;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
 
 import org.openmbee.mms.core.objects.ElementsRequest;
 import org.openmbee.mms.core.objects.ElementsResponse;
@@ -16,10 +23,11 @@
 import org.openmbee.mms.core.services.NodeService;
 import org.openmbee.mms.core.pubsub.EmbeddedHookService;
 import org.openmbee.mms.crud.hooks.ElementUpdateHook;
-import org.openmbee.mms.crud.services.DefaultCommitService;
+import org.openmbee.mms.json.ElementJson;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.http.HttpStatus;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpEntity;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
@@ -44,6 +52,9 @@ public class ElementsController extends BaseController {
     private EmbeddedHookService embeddedHookService;
     private CommitService commitService;
 
+    @Value("${mms.stream.batch.size:5000}")
+    private int streamLimit;
+
     @Autowired
     public void setEmbeddedHookService(EmbeddedHookService embeddedHookService) {
         this.embeddedHookService = embeddedHookService;
@@ -54,6 +65,8 @@ public void setCommitService(@Qualifier("defaultCommitService")CommitService com
         this.commitService = commitService;
     }
 
+    private static final JsonFactory jfactory = new JsonFactory();
+
     @GetMapping
     @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_READ', true)")
     @ApiResponse(responseCode = "200", content = {
@@ -112,6 +125,55 @@ public ElementsResponse createOrUpdateElements(
         }
         throw new BadRequestException(response.addMessage("Empty"));
     }
+    /*
+    @PostMapping(value = "/stream", consumes = MediaType.APPLICATION_JSON_VALUE)
+    @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_EDIT_CONTENT', false)")
+    */
+    public ResponseEntity<StreamingResponseBody> createOrUpdateElementsStream(
+        @PathVariable String projectId,
+        @PathVariable String refId,
+        @RequestParam(required = false) Map<String, String> params,
+        @Parameter(hidden = true) @RequestHeader(value = "Accept", defaultValue = "application/json") String accept,
+        Authentication auth,
+        HttpEntity<byte[]> requestEntity) {
+
+        String commitId = UUID.randomUUID().toString(); // Generate a commitId from the start
+        params.put("commitId", commitId);
+        ElementsRequest req = new ElementsRequest();
+        List<ElementJson> elements = new ArrayList<>();
+
+        InputStream stream = new ByteArrayInputStream(Objects.requireNonNull(requestEntity.getBody()));
+        StreamingResponseBody response =  outputStream -> {
+            ObjectMapper om = new ObjectMapper();
+            try (JsonParser parser = jfactory.createParser(stream)) {
+                if(parser.nextToken() != JsonToken.START_OBJECT) {
+                    throw new BadRequestException("Expected an object");
+                }
+                while (parser.nextToken() != JsonToken.END_OBJECT) {
+                    logger.debug("Current Token: " + parser.getCurrentName());
+                    if (parser.nextToken() == JsonToken.START_ARRAY && "elements".equals(parser.getCurrentName())) {
+                        logger.debug("Found Array: " + parser.getCurrentName());
+                        while (parser.nextToken() != JsonToken.END_OBJECT) {
+                            ElementJson node = om.readValue(parser, ElementJson.class);
+                            elements.add(node);
+                            //outputStream.write(node.getType().getBytes(StandardCharsets.UTF_8));
+                            //outputStream.write(node.get("id").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    }
+                }
+                req.setElements(elements);
+                if (!req.getElements().isEmpty()) {
+                    NodeService nodeService = getNodeService(projectId);
+                    nodeService.createOrUpdate(projectId, refId, req, params, auth.getName());
+                }
+            } catch (IOException e) {
+                logger.debug("Error in stream handling: ", e);
+            }
+        };
+        return ResponseEntity.ok()
+            .header("Content-Type", accept.equals("application/x-ndjson") ? accept : "application/json")
+            .body(response);
+    }
 
     @PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
     @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_READ', true)")

crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java

@@ -1,10 +1,7 @@
 package org.openmbee.mms.crud.controllers.projects;
 
 import io.swagger.v3.oas.annotations.tags.Tag;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
+
 import org.openmbee.mms.core.config.ContextHolder;
 import org.openmbee.mms.core.config.Privileges;
 import org.openmbee.mms.core.config.ProjectSchemas;
@@ -35,6 +32,12 @@
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
 @RestController
 @RequestMapping("/projects")
 @Tag(name = "Projects")
@@ -55,10 +58,10 @@ public ProjectsController(ProjectDAO projectRepository, ProjectIndex projectInde
     }
 
     @GetMapping
-    public ProjectsResponse getAllProjects(Authentication auth) {
-
+    public ProjectsResponse getAllProjects(Authentication auth,
+                                           @RequestParam(required = false) String orgId) {
         ProjectsResponse response = new ProjectsResponse();
-        List<Project> allProjects = projectRepository.findAll();
+        List<Project> allProjects = orgId != null ? projectRepository.findAllByOrgId(orgId) : projectRepository.findAll();
         for (Project proj : allProjects) {
             if (mss.hasProjectPrivilege(auth, proj.getProjectId(), Privileges.PROJECT_READ.name(), true)) {
                 ContextHolder.setContext(proj.getProjectId());

crud/src/main/java/org/openmbee/mms/crud/services/DefaultBranchService.java

@@ -150,7 +150,7 @@ public RefJson createBranch(String projectId, RefJson branch) {
             Optional<Commit> parentCommit = commitRepository.findLatestByRef(refOption.get());
             parentCommit.ifPresent(parent -> {
                 b.setParentCommit(parent.getId());
-                branch.setParentCommitId(parent.getDocId()); //commit id is same as its docId
+                branch.setParentCommitId(parent.getCommitId()); //commit id is same as its docId
             });
         }
 
@@ -172,6 +172,7 @@ public RefJson createBranch(String projectId, RefJson branch) {
             return branch;
         } catch (Exception e) {
             logger.error("Couldn't create branch: {}", branch.getId(), e);
+            //TODO should clean up any created tables/rows?
             throw new InternalErrorException(e);
         }
     }

crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java

@@ -77,8 +77,10 @@ public CommitsResponse getRefCommits(String projectId, String refId, Map<String,
                 CommitJson json = new CommitJson();
                 json.setCreated(Formats.FORMATTER.format(c.getTimestamp()));
                 json.setCreator(c.getCreator());
-                json.setId(c.getDocId());
+                json.setId(c.getCommitId());
                 json.setComment(c.getComment());
+                json.setRefId(c.getBranchId());
+                json.setProjectId(projectId);
                 resJson.add(json);
             }
         }, () -> {
@@ -114,7 +116,7 @@ public CommitsResponse getElementCommits(String projectId, String refId, String
         List<Commit> refCommits = commitRepository.findByRefAndTimestampAndLimit(ref.get(), null, 0);
         Set<String> commitIds = new HashSet<>();
         for (Commit commit: refCommits) {
-            commitIds.add(commit.getDocId());
+            commitIds.add(commit.getCommitId());
         }
         res.getCommits().addAll(commitIndex.elementHistory(elementId, commitIds));
         return res;

crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java

@@ -189,10 +189,11 @@ public ElementsResponse createOrUpdate(String projectId, String refId, ElementsR
         ContextHolder.setContext(projectId, refId);
         boolean overwriteJson = Boolean.parseBoolean(params.get("overwrite"));
         nodePostHelper.setPreserveTimestamps(Boolean.parseBoolean(params.get("preserveTimestamps")));
+        String commitId = params.get("commitId");
 
         NodeChangeInfo info = nodePostHelper
             .processPostJson(req.getElements(), overwriteJson,
-                createCommit(user, refId, projectId, req), this);
+                createCommit(user, refId, projectId, req, commitId), this);
 
         commitChanges(info);
 
@@ -220,16 +221,23 @@ protected void commitChanges(NodeChangeInfo info) {
                 }
                 this.nodeIndex.removeFromRef(info.getOldDocIds());
 
-                Commit commit = new Commit();
-                commit.setBranchId(cmjs.getRefId());
-                commit.setCommitType(CommitType.COMMIT);
-                commit.setCreator(cmjs.getCreator());
-                commit.setDocId(cmjs.getId());
-                commit.setTimestamp(now);
-                commit.setComment(cmjs.getComment());
-
-                this.commitRepository.save(commit);
+                Optional<Commit> existing = this.commitRepository.findByCommitId(cmjs.getId());
+                existing.ifPresentOrElse(
+                    current -> {
+                        this.logger.debug(String.format("Commit object %s already exists. Skipping record creation.", current.getCommitId()));
+                    },
+                    () -> {
+                        Commit commit = new Commit();
+                        commit.setCommitId(cmjs.getId());
+                        commit.setBranchId(cmjs.getRefId());
+                        commit.setCommitType(CommitType.COMMIT);
+                        commit.setCreator(cmjs.getCreator());
+                        commit.setTimestamp(now);
+                        commit.setComment(cmjs.getComment());
+                        this.commitRepository.save(commit);
+                    });
                 this.commitIndex.index(cmjs);
+
                 this.nodeRepository.getTransactionManager().commit(status);
             } catch (Exception e) {
                 logger.error("commitChanges error: ", e);
@@ -282,7 +290,7 @@ public ElementsResponse delete(String projectId, String refId, ElementsRequest r
         ContextHolder.setContext(projectId, refId);
 
         NodeChangeInfo info = nodeDeleteHelper
-            .processDeleteJson(req.getElements(), createCommit(user, refId, projectId, req),
+            .processDeleteJson(req.getElements(), createCommit(user, refId, projectId, req, null),
                 this);
         ElementsResponse response = new ElementsResponse();
 
@@ -294,13 +302,18 @@ public ElementsResponse delete(String projectId, String refId, ElementsRequest r
     }
 
     private CommitJson createCommit(String creator, String refId, String projectId,
-        ElementsRequest req) {
+        ElementsRequest req, String commitId) {
         CommitJson cmjs = new CommitJson();
         cmjs.setCreator(creator);
         cmjs.setComment(req.getComment());
         cmjs.setSource(req.getSource());
         cmjs.setRefId(refId);
         cmjs.setProjectId(projectId);
+
+        if (commitId != null && !commitId.isEmpty()) {
+            cmjs.setId(commitId);
+        }
+
         return cmjs;
     }
 

crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java

@@ -113,7 +113,7 @@ public NodeGetInfo processGetJson(List<ElementJson> elements, Instant time, Node
         if (ref.isPresent()) {
             Optional<Commit> c = commitRepository.findLatestByRef(ref.get());
             if (c.isPresent()) {
-                return processGetJson(elements, c.get().getDocId(), service);
+                return processGetJson(elements, c.get().getCommitId(), service);
             } else {
                 throw new BadRequestException("invalid time");
             }
@@ -148,7 +148,7 @@ public List<ElementJson> processGetAll(Instant time, NodeService service) {
         if (ref.isPresent()) {
             Optional<Commit> c = commitRepository.findByRefAndTimestamp(ref.get(), time);
             if (c.isPresent()) {
-                result.addAll(processGetAll(c.get().getDocId(), service));
+                result.addAll(processGetAll(c.get().getCommitId(), service));
             } else {
                 throw new BadRequestException("invalid time");
             }
@@ -167,7 +167,7 @@ protected List<String> getRefCommitIds(Instant time) {
         ref.ifPresent(current -> {
             List<Commit> refCommits = commitRepository.findByRefAndTimestampAndLimit(current, time, 0);
             for (Commit c : refCommits) {
-                commitIds.add(c.getDocId());
+                commitIds.add(c.getCommitId());
             }
         });
         return commitIds;

crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java

@@ -65,8 +65,13 @@ public void setBranchRepository(BranchDAO branchRepository) {
     }
 
     public void initCommitJson(CommitJson cmjs, Instant now) {
-        cmjs.setId(UUID.randomUUID().toString());
-        cmjs.setDocId(cmjs.getId());
+        if (cmjs.getId() == null || cmjs.getId().isEmpty()) {
+            cmjs.setId(UUID.randomUUID().toString());
+            cmjs.setDocId(cmjs.getId());
+        }
+        if (cmjs.getDocId() == null || cmjs.getDocId().isEmpty()) {
+            cmjs.setDocId(UUID.randomUUID().toString());
+        }
         cmjs.setCreated(formatter.format(now));
         cmjs.setAdded(new ArrayList<>());
         cmjs.setDeleted(new ArrayList<>());
@@ -130,7 +135,7 @@ public void processElementAdded(ElementJson e, Node n, NodeChangeInfo info) {
         cmjs.getAdded().add(newObj);
 
         n.setNodeId(e.getId());
-        n.setInitialCommit(e.getDocId());
+        n.setInitialCommit(e.getCommitId());
     }
 
     public void processElementUpdated(ElementJson e, Node n, NodeChangeInfo info) {

data/src/main/java/org/openmbee/mms/data/domains/scoped/Branch.java

@@ -19,11 +19,13 @@ public class Branch {
 
     private String description;
 
-    @Column(unique = true)
+    @Column(unique = true, length = 512)
     private String branchId;
     private String docId;
+    @Column(length = 512)
     private String branchName;
 
+    @Column(length = 512)
     private String parentRefId;
     private Long parentCommit;