diff --git a/authenticator/src/main/java/org/openmbee/mms/authenticator/controllers/AuthenticationController.java b/authenticator/src/main/java/org/openmbee/mms/authenticator/controllers/AuthenticationController.java index 8cb5a1bcb..2302adb72 100644 --- a/authenticator/src/main/java/org/openmbee/mms/authenticator/controllers/AuthenticationController.java +++ b/authenticator/src/main/java/org/openmbee/mms/authenticator/controllers/AuthenticationController.java @@ -53,7 +53,7 @@ public JwtAuthenticationResponse createAuthenticationToken( @GetMapping(value = "/authentication") @PreAuthorize("isAuthenticated()") - public JwtAuthenticationResponse createAuthenticationToken() { + public JwtAuthenticationResponse getAuthenticationToken() { final Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); if (authentication != null && authentication.getPrincipal() instanceof UserDetails) { final UserDetails userDetails = (UserDetails) authentication.getPrincipal(); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/BadRequestException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/BadRequestException.java index 063c3a46a..fa6948e63 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/BadRequestException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/BadRequestException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class BadRequestException extends SdvcException { +public class BadRequestException extends MMSException { public BadRequestException(Object body) { super(HttpStatus.BAD_REQUEST, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/ConflictException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/ConflictException.java index 65b191f7d..ecc945d74 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/ConflictException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/ConflictException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class ConflictException extends SdvcException { +public class ConflictException extends MMSException { public ConflictException(Object body) { super(HttpStatus.CONFLICT, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/DeletedException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/DeletedException.java index 358658ea7..f7c1e0b77 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/DeletedException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/DeletedException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class DeletedException extends SdvcException { +public class DeletedException extends MMSException { public DeletedException(Object body) { super(HttpStatus.GONE, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/ForbiddenException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/ForbiddenException.java index 0f339a332..b6b2c7b92 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/ForbiddenException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/ForbiddenException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class ForbiddenException extends SdvcException { +public class ForbiddenException extends MMSException { public ForbiddenException(Object body) { super(HttpStatus.FORBIDDEN, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/InternalErrorException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/InternalErrorException.java index ff26f1837..0c88cd64f 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/InternalErrorException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/InternalErrorException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class InternalErrorException extends SdvcException { +public class InternalErrorException extends MMSException { public InternalErrorException(Object body) { super(HttpStatus.INTERNAL_SERVER_ERROR, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/SdvcException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java similarity index 75% rename from core/src/main/java/org/openmbee/mms/core/exceptions/SdvcException.java rename to core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java index 702ffe739..bc45a655f 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/SdvcException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/MMSException.java @@ -2,20 +2,20 @@ import org.springframework.http.HttpStatus; -public abstract class SdvcException extends RuntimeException { +public abstract class MMSException extends RuntimeException { private HttpStatus code; private Object messageObject; - public SdvcException() { + public MMSException() { } - public SdvcException(HttpStatus code, Object messageObject) { + public MMSException(HttpStatus code, Object messageObject) { this.code = code; this.messageObject = messageObject; } - public SdvcException(int code, Object messageObject) { + public MMSException(int code, Object messageObject) { this.code = HttpStatus.resolve(code); this.messageObject = messageObject; } diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/NotFoundException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/NotFoundException.java index dc1face62..673e0c07b 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/NotFoundException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/NotFoundException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class NotFoundException extends SdvcException { +public class NotFoundException extends MMSException { public NotFoundException(Object body) { super(HttpStatus.NOT_FOUND, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/NotModifiedException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/NotModifiedException.java index ebab0134b..817d88e86 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/NotModifiedException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/NotModifiedException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class NotModifiedException extends SdvcException { +public class NotModifiedException extends MMSException { public NotModifiedException(Object body) { super(HttpStatus.NOT_MODIFIED, body); diff --git a/core/src/main/java/org/openmbee/mms/core/exceptions/UnauthorizedException.java b/core/src/main/java/org/openmbee/mms/core/exceptions/UnauthorizedException.java index 9e76af2c6..5c54ccb96 100644 --- a/core/src/main/java/org/openmbee/mms/core/exceptions/UnauthorizedException.java +++ b/core/src/main/java/org/openmbee/mms/core/exceptions/UnauthorizedException.java @@ -2,7 +2,7 @@ import org.springframework.http.HttpStatus; -public class UnauthorizedException extends SdvcException { +public class UnauthorizedException extends MMSException { public UnauthorizedException(Object body) { super(HttpStatus.UNAUTHORIZED, body); diff --git a/core/src/main/java/org/openmbee/mms/core/services/NodeService.java b/core/src/main/java/org/openmbee/mms/core/services/NodeService.java index cde64507a..7db8b19d4 100644 --- a/core/src/main/java/org/openmbee/mms/core/services/NodeService.java +++ b/core/src/main/java/org/openmbee/mms/core/services/NodeService.java @@ -1,5 +1,7 @@ package org.openmbee.mms.core.services; +import java.io.IOException; +import java.io.OutputStream; import java.util.Map; import org.openmbee.mms.core.objects.ElementsRequest; @@ -9,6 +11,8 @@ public interface NodeService { + void readAsStream(String projectId, String refId, Map params, OutputStream output, String accept) throws IOException; + ElementsResponse read(String projectId, String refId, String id, Map params); ElementsResponse read(String projectId, String refId, ElementsRequest req, Map params); diff --git a/crud/src/main/java/org/openmbee/mms/crud/config/ExceptionHandlerConfig.java b/crud/src/main/java/org/openmbee/mms/crud/config/ExceptionHandlerConfig.java index 938cd67ed..348094593 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/config/ExceptionHandlerConfig.java +++ b/crud/src/main/java/org/openmbee/mms/crud/config/ExceptionHandlerConfig.java @@ -1,6 +1,7 @@ package org.openmbee.mms.crud.config; -import org.openmbee.mms.core.exceptions.SdvcException; +import java.util.Map; +import org.openmbee.mms.core.exceptions.MMSException; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.http.HttpHeaders; @@ -14,8 +15,11 @@ @Order(Ordered.LOWEST_PRECEDENCE) public class ExceptionHandlerConfig extends ResponseEntityExceptionHandler { - @ExceptionHandler(value = { SdvcException.class }) - protected ResponseEntity handleSdvcException(SdvcException ex, WebRequest request) { + @ExceptionHandler(value = { MMSException.class }) + protected ResponseEntity handleMMSException(MMSException ex, WebRequest request) { + if (ex.getMessageObject() instanceof String) { + ex.setMessageObject(Map.of("message", ex.getMessageObject())); + } return handleExceptionInternal(ex, ex.getMessageObject(), new HttpHeaders(), ex.getCode(), request); } diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/branches/BranchesController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/branches/BranchesController.java index d62f78c65..60e61318b 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/branches/BranchesController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/branches/BranchesController.java @@ -4,7 +4,7 @@ import java.util.UUID; import org.openmbee.mms.core.config.Privileges; import org.openmbee.mms.core.exceptions.BadRequestException; -import org.openmbee.mms.core.exceptions.SdvcException; +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.core.objects.RefsRequest; import org.openmbee.mms.core.objects.RefsResponse; import org.openmbee.mms.core.objects.Rejection; @@ -97,7 +97,7 @@ public RefsResponse createRefs( permissionService.initBranchPerms(projectId, branch.getId(), true, auth.getName()); response.getRefs().add(res); - } catch (SdvcException e) { + } catch (MMSException e) { response.addRejection(new Rejection(branch, e.getCode().value(), e.getMessageObject().toString())); } } diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java index 311696ba3..f711affd7 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/elements/ElementsController.java @@ -1,18 +1,27 @@ package org.openmbee.mms.crud.controllers.elements; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import java.util.Map; import org.openmbee.mms.core.objects.ElementsRequest; import org.openmbee.mms.core.objects.ElementsResponse; +import org.openmbee.mms.core.services.CommitService; import org.openmbee.mms.crud.controllers.BaseController; import org.openmbee.mms.core.exceptions.BadRequestException; import org.openmbee.mms.core.services.NodeService; import org.openmbee.mms.core.pubsub.EmbeddedHookService; import org.openmbee.mms.crud.hooks.ElementUpdateHook; +import org.openmbee.mms.crud.services.DefaultCommitService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.Authentication; import org.springframework.web.bind.annotation.DeleteMapping; @@ -21,9 +30,11 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; @RestController @RequestMapping("/projects/{projectId}/refs/{refId}/elements") @@ -31,22 +42,39 @@ public class ElementsController extends BaseController { private EmbeddedHookService embeddedHookService; + private CommitService commitService; @Autowired public void setEmbeddedHookService(EmbeddedHookService embeddedHookService) { this.embeddedHookService = embeddedHookService; } + @Autowired + public void setCommitService(@Qualifier("defaultCommitService")CommitService commitService) { + this.commitService = commitService; + } + @GetMapping @PreAuthorize("@mss.hasBranchPrivilege(authentication, #projectId, #refId, 'BRANCH_READ', true)") - public ElementsResponse getAllElements( + @ApiResponse(responseCode = "200", content = { + @Content(mediaType = "application/json", schema = @Schema(implementation = ElementsResponse.class)), + @Content(mediaType = "application/x-ndjson") + }) + public ResponseEntity getAllElements( @PathVariable String projectId, @PathVariable String refId, @RequestParam(required = false) String commitId, - @RequestParam(required = false) Map params) { + @RequestParam(required = false) Map params, + @Parameter(hidden = true) @RequestHeader(value = "Accept", defaultValue = "application/json") String accept) { NodeService nodeService = getNodeService(projectId); - return nodeService.read(projectId, refId, "", params); + if (commitId != null && !commitId.isEmpty()) { + commitService.getCommit(projectId, commitId); //check commit exists + } + StreamingResponseBody stream = outputStream -> nodeService.readAsStream(projectId, refId, params, outputStream, accept); + return ResponseEntity.ok() + .header("Content-Type", accept.equals("application/x-ndjson") ? accept : "application/json") + .body(stream); } @GetMapping(value = "/{elementId}", produces = MediaType.APPLICATION_JSON_VALUE) @@ -58,7 +86,6 @@ public ElementsResponse getElement( @RequestParam(required = false) String commitId, @RequestParam(required = false) Map params) { - NodeService nodeService = getNodeService(projectId); ElementsResponse res = nodeService.read(projectId, refId, elementId, params); handleSingleResponse(res); diff --git a/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java b/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java index 3846f6d13..ebf197ee0 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java +++ b/crud/src/main/java/org/openmbee/mms/crud/controllers/projects/ProjectsController.java @@ -10,7 +10,7 @@ import org.openmbee.mms.core.config.ProjectSchemas; import org.openmbee.mms.core.dao.ProjectDAO; import org.openmbee.mms.core.dao.ProjectIndex; -import org.openmbee.mms.core.exceptions.SdvcException; +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.core.objects.ProjectsRequest; import org.openmbee.mms.core.objects.ProjectsResponse; import org.openmbee.mms.core.exceptions.DeletedException; @@ -160,7 +160,7 @@ public ProjectsResponse createOrUpdateProjects( permissionService.setProjectInherit(true, json.getProjectId()); } } - } catch (SdvcException ex) { + } catch (MMSException ex) { response.addRejection(new Rejection(json, ex.getCode().value(), ex.getMessageObject().toString())); continue; } diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java index 735e23ed1..863a9527a 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultCommitService.java @@ -93,16 +93,12 @@ public CommitsResponse getRefCommits(String projectId, String refId, Map commit = commitIndex.findById(commitId); - if (commit.isPresent()) { - res.getCommits().add(commit.get()); - } else { - throw new NotFoundException(res); - } - } catch (Exception e) { - e.printStackTrace(); - throw new InternalErrorException(e); + + Optional commit = commitIndex.findById(commitId); + if (commit.isPresent()) { + res.getCommits().add(commit.get()); + } else { + throw new NotFoundException("Commit not found"); } return res; } @@ -111,21 +107,16 @@ public CommitsResponse getCommit(String projectId, String commitId) { public CommitsResponse getElementCommits(String projectId, String refId, String elementId, Map params) { ContextHolder.setContext(projectId); CommitsResponse res = new CommitsResponse(); - try { - Optional ref = branchRepository.findByBranchId(refId); - if (!ref.isPresent()) { - throw new NotFoundException("Branch not found"); - } - List refCommits = commitRepository.findByRefAndTimestampAndLimit(ref.get(), null, 0); - Set commitIds = new HashSet<>(); - for (Commit commit: refCommits) { - commitIds.add(commit.getDocId()); - } - res.getCommits().addAll(commitIndex.elementHistory(elementId, commitIds)); - } catch (Exception e) { - e.printStackTrace(); - throw new InternalErrorException(e); + Optional ref = branchRepository.findByBranchId(refId); + if (!ref.isPresent()) { + throw new NotFoundException("Branch not found"); + } + List refCommits = commitRepository.findByRefAndTimestampAndLimit(ref.get(), null, 0); + Set commitIds = new HashSet<>(); + for (Commit commit: refCommits) { + commitIds.add(commit.getDocId()); } + res.getCommits().addAll(commitIndex.elementHistory(elementId, commitIds)); return res; } diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java index a823df6ca..f660e6284 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java @@ -1,8 +1,18 @@ package org.openmbee.mms.crud.services; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.openmbee.mms.core.exceptions.BadRequestException; import org.openmbee.mms.core.objects.EventObject; import org.openmbee.mms.core.services.EventService; import org.openmbee.mms.core.services.NodeChangeInfo; @@ -24,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; @@ -33,6 +44,10 @@ @Service("defaultNodeService") public class DefaultNodeService implements NodeService { + @Value("${mms.stream.batch.size:100000}") + protected int streamLimit; + protected final ObjectMapper objectMapper = new ObjectMapper(); + protected final Logger logger = LoggerFactory.getLogger(getClass()); protected NodeDAO nodeRepository; @@ -86,6 +101,50 @@ public void setEventPublisher(Collection eventPublisher) { this.eventPublisher = eventPublisher; } + @Override + public void readAsStream(String projectId, String refId, + Map params, OutputStream stream, String accept) throws IOException { + + String commitId = params.getOrDefault("commitId", null); + ContextHolder.setContext(projectId, refId); + List nodes; + if (commitId != null && !commitId.isEmpty()) { + if (!commitRepository.findByCommitId(commitId).isPresent()) { + throw new BadRequestException("commit id is invalid"); + } + nodes = nodeRepository.findAll(); + } else { + nodes = nodeRepository.findAllByDeleted(false); + } + String separator = "\n"; + if (!"application/x-ndjson".equals(accept)) { + stream.write("{\"elements\":[".getBytes(StandardCharsets.UTF_8)); + separator = ","; + } + final String sep = separator; + AtomicInteger counter = new AtomicInteger(); + batches(nodes, streamLimit).forEach(ns -> { + try { + if (counter.get() == 0) { + counter.getAndIncrement(); + } else { + stream.write(sep.getBytes(StandardCharsets.UTF_8)); + } + Collection result = nodeGetHelper.processGetJsonFromNodes(ns, commitId, this) + .getActiveElementMap().values(); + stream.write(result.stream().map(this::toJson).collect(Collectors.joining(sep)) + .getBytes(StandardCharsets.UTF_8)); + } catch (IOException ioe) { + logger.error("Error writing to stream", ioe); + } + }); + if (!"application/x-ndjson".equals(accept)) { + stream.write("]}".getBytes(StandardCharsets.UTF_8)); + } else { + stream.write("\n".getBytes(StandardCharsets.UTF_8)); + } + } + @Override public ElementsResponse read(String projectId, String refId, String id, Map params) { @@ -244,4 +303,18 @@ private CommitJson createCommit(String creator, String refId, String projectId, cmjs.setProjectId(projectId); return cmjs; } + + protected static Stream> batches(List source, int length) { + return IntStream.iterate(0, i -> i < source.size(), i -> i + length) + .mapToObj(i -> source.subList(i, Math.min(i + length, source.size()))); + } + + protected String toJson(ElementJson elementJson) { + try { + return objectMapper.writeValueAsString(elementJson); + } catch (JsonProcessingException e) { + logger.error("Error in toJson: ", e); + } + return ""; + } } diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java index 0cc07df35..c043ddd89 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeGetHelper.java @@ -19,9 +19,17 @@ @Service public class NodeGetHelper extends NodeOperation { + public NodeGetInfo processGetJsonFromNodes(List nodes, NodeService service) { + NodeGetInfo info = initInfoFromNodes(nodes, null); + return processLatest(info, service); + } + public NodeGetInfo processGetJson(List elements, NodeService service) { NodeGetInfo info = initInfo(elements, null); + return processLatest(info, service); + } + private NodeGetInfo processLatest(NodeGetInfo info, NodeService service) { for (String nodeId : info.getReqElementMap().keySet()) { if (!existingNodeContainsNodeId(info, nodeId)) { continue; @@ -39,19 +47,29 @@ public NodeGetInfo processGetJson(List elements, NodeService servic return info; } + public NodeGetInfo processGetJsonFromNodes(List nodes, String commitId, NodeService service) { + if (commitId == null || commitId.isEmpty()) { + return processGetJsonFromNodes(nodes, service); + } + NodeGetInfo info = initInfoFromNodes(nodes, null); + return processCommit(info, commitId, service); + } + public NodeGetInfo processGetJson(List elements, String commitId, NodeService service) { if (commitId == null || commitId.isEmpty()) { return processGetJson(elements, service); } + NodeGetInfo info = initInfo(elements, null); //gets all current nodes + return processCommit(info, commitId, service); + } + private NodeGetInfo processCommit(NodeGetInfo info, String commitId, NodeService service) { Optional commit = commitRepository.findByCommitId(commitId); if (!commit.isPresent() ) { throw new BadRequestException("commitId is invalid"); } Instant time = commit.get().getTimestamp(); //time of commit List refCommitIds = null; //get it later if needed - - NodeGetInfo info = initInfo(elements, null); //gets all current nodes for (String nodeId : info.getReqElementMap().keySet()) { if (!existingNodeContainsNodeId(info, nodeId)) { // nodeId not found continue; diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java index a60dd75fd..284c5f124 100644 --- a/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java +++ b/crud/src/main/java/org/openmbee/mms/crud/services/NodeOperation.java @@ -74,15 +74,14 @@ public void initCommitJson(CommitJson cmjs, Instant now) { cmjs.setType("Commit"); } - public NodeChangeInfo initInfo(List elements, CommitJson cmjs) { - + public NodeChangeInfo initInfoFromNodes(List existingNodes, CommitJson cmjs) { Set indexIds = new HashSet<>(); - Map reqElementMap = convertJsonToMap(elements); - List existingNodes = nodeRepository.findAllByNodeIds(reqElementMap.keySet()); Map existingNodeMap = new HashMap<>(); + Map reqElementMap = new HashMap<>(); for (Node node : existingNodes) { indexIds.add(node.getDocId()); existingNodeMap.put(node.getNodeId(), node); + reqElementMap.put(node.getNodeId(), new ElementJson().setId(node.getNodeId())); } // bulk read existing elements in elastic List existingElements = nodeIndex.findAllById(indexIds); @@ -109,6 +108,14 @@ public NodeChangeInfo initInfo(List elements, CommitJson cmjs) { return info; } + public NodeChangeInfo initInfo(List elements, CommitJson cmjs) { + Map reqElementMap = convertJsonToMap(elements); + List existingNodes = nodeRepository.findAllByNodeIds(reqElementMap.keySet()); + NodeChangeInfo info = initInfoFromNodes(existingNodes, cmjs); + info.setReqElementMap(reqElementMap); + return info; + } + public void processElementAdded(ElementJson e, Node n, NodeChangeInfo info) { CommitJson cmjs = info.getCommitJson(); processElementAddedOrUpdated(e, n, info); diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java index b33a70d26..c574a955e 100644 --- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java +++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java @@ -42,12 +42,18 @@ public abstract class BaseElasticDAOImpl> { private final Logger logger = LoggerFactory.getLogger(getClass()); - @Value("${elasticsearch.limit.result}") + @Value("${elasticsearch.limit.result:10000}") protected int resultLimit; - @Value("${elasticsearch.limit.term}") + + @Value("${elasticsearch.limit.term:1000}") protected int termLimit; - @Value("${elasticsearch.limit.get}") + + @Value("${elasticsearch.limit.get:100000}") protected int getLimit; + + @Value("${elasticsearch.limit.index:5000}") + protected int bulkLimit; + protected static int readTimeout = 1000000000; protected RestHighLevelClient client; private static final RequestOptions REQUEST_OPTIONS; @@ -203,11 +209,11 @@ public E update(String index, BaseJson json) { return response; } - private static BulkProcessor getBulkProcessor(RestHighLevelClient client) { + private BulkProcessor getBulkProcessor(RestHighLevelClient client) { return getBulkProcessor(client, null); } - private static BulkProcessor getBulkProcessor(RestHighLevelClient client, BulkProcessor.Listener listener) { + private BulkProcessor getBulkProcessor(RestHighLevelClient client, BulkProcessor.Listener listener) { if (listener == null) { listener = new BulkProcessor.Listener() { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -217,6 +223,13 @@ public void beforeBulk(long executionId, BulkRequest request) { @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + if (response.hasFailures()) { + response.iterator().forEachRemaining(action -> { + if (action.isFailed()) { + logger.error("Error in bulk processing: {}", action.getFailureMessage()); + } + }); + } } @Override @@ -227,7 +240,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) } BulkProcessor.Builder bpBuilder = BulkProcessor.builder((request, bulkListener) -> client .bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); - bpBuilder.setBulkActions(5000); + bpBuilder.setBulkActions(bulkLimit); bpBuilder.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)); bpBuilder.setConcurrentRequests(1); bpBuilder.setFlushInterval(TimeValue.timeValueSeconds(5)); diff --git a/elastic/src/main/resources/application.properties.example b/elastic/src/main/resources/application.properties.example index 39cfa22be..380f79b2a 100644 --- a/elastic/src/main/resources/application.properties.example +++ b/elastic/src/main/resources/application.properties.example @@ -3,4 +3,6 @@ elasticsearch.port=9200 elasticsearch.http=http elasticsearch.limit.result=100 elasticsearch.limit.term=100 -elasticsearch.limit.scrollTimeout=1000 \ No newline at end of file +elasticsearch.limit.scrollTimeout=1000 +elasticsearch.limit.get=100000 +elasticsearch.limit.index=5000 \ No newline at end of file diff --git a/example/src/main/resources/application-test.properties b/example/src/main/resources/application-test.properties index 2cb508f36..6d2cd5455 100644 --- a/example/src/main/resources/application-test.properties +++ b/example/src/main/resources/application-test.properties @@ -1,6 +1,8 @@ # See authenticator module for example configuration -sdvc.admin.username=test -sdvc.admin.password=test +mms.admin.username=test +mms.admin.password=test + +mms.stream.batch.size=100000 jwt.secret=12345678901234567890123456789012 jwt.expiration=86400 @@ -52,6 +54,7 @@ elasticsearch.limit.result=10000 elasticsearch.limit.term=1000 elasticsearch.limit.scrollTimeout=1000 elasticsearch.limit.get=10000 +elasticsearch.limit.index=5000 #Configuration for TWC #port is for REST interface diff --git a/example/src/main/resources/application.properties.example b/example/src/main/resources/application.properties.example index 6d0d11e24..001b74fce 100644 --- a/example/src/main/resources/application.properties.example +++ b/example/src/main/resources/application.properties.example @@ -1,6 +1,8 @@ # See authenticator module for example configuration -sdvc.admin.username=test -sdvc.admin.password=test +mms.admin.username=test +mms.admin.password=test + +mms.stream.batch.size=100000 jwt.secret=make_me_something_really_long jwt.expiration=86400 @@ -50,6 +52,7 @@ elasticsearch.limit.result=10000 elasticsearch.limit.term=1000 elasticsearch.limit.scrollTimeout=1000 elasticsearch.limit.get=100000 +elasticsearch.limit.index=5000 #Configuration for TWC #port is for REST interface diff --git a/gradle.properties b/gradle.properties index 40226acdf..61d35a417 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=4.0.0-b3 +version=4.0.0-b4 group=org.openmbee.mms springBootVersion=2.2.6.RELEASE diff --git a/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java b/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java index 5116e9a3f..b96c10577 100644 --- a/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java +++ b/localuser/src/main/java/org/openmbee/mms/localuser/config/AuthProviderConfig.java @@ -20,9 +20,9 @@ public class AuthProviderConfig { private UserDetailsServiceImpl userDetailsService; private PasswordEncoder passwordEncoder; - @Value("${sdvc.admin.username}") + @Value("${mms.admin.username}") private String adminUsername; - @Value("${sdvc.admin.password}") + @Value("${mms.admin.password}") private String adminPassword; @Autowired diff --git a/localuser/src/main/resources/application.properties.example b/localuser/src/main/resources/application.properties.example index b24af9ec4..54d46a36f 100644 --- a/localuser/src/main/resources/application.properties.example +++ b/localuser/src/main/resources/application.properties.example @@ -1,2 +1,2 @@ -sdvc.admin.username= -sdvc.admin.password= \ No newline at end of file +mms.admin.username= +mms.admin.password= \ No newline at end of file diff --git a/permissions/src/main/java/org/openmbee/mms/permissions/PermissionsLookupController.java b/permissions/src/main/java/org/openmbee/mms/permissions/PermissionsLookupController.java index e52810019..1354ce64d 100644 --- a/permissions/src/main/java/org/openmbee/mms/permissions/PermissionsLookupController.java +++ b/permissions/src/main/java/org/openmbee/mms/permissions/PermissionsLookupController.java @@ -3,7 +3,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import java.util.ArrayList; import java.util.List; -import org.openmbee.mms.core.exceptions.SdvcException; +import org.openmbee.mms.core.exceptions.MMSException; import org.openmbee.mms.core.objects.Rejection; import org.openmbee.mms.core.security.MethodSecurityService; import org.openmbee.mms.permissions.objects.PermissionLookup; @@ -59,7 +59,7 @@ public PermissionLookupResponse lookupPermissions( if (!result) { res.setAllPassed(false); } - } catch (SdvcException e) { + } catch (MMSException e) { res.addRejection(new Rejection(lookup, e.getCode().value(), e.getMessage())); res.setAllPassed(false); } catch (Exception e) { diff --git a/permissions/src/main/java/org/openmbee/mms/permissions/exceptions/PermissionException.java b/permissions/src/main/java/org/openmbee/mms/permissions/exceptions/PermissionException.java index 45a320ecf..5b24c67d5 100644 --- a/permissions/src/main/java/org/openmbee/mms/permissions/exceptions/PermissionException.java +++ b/permissions/src/main/java/org/openmbee/mms/permissions/exceptions/PermissionException.java @@ -1,9 +1,9 @@ package org.openmbee.mms.permissions.exceptions; -import org.openmbee.mms.core.exceptions.SdvcException; +import org.openmbee.mms.core.exceptions.MMSException; import org.springframework.http.HttpStatus; -public class PermissionException extends SdvcException { +public class PermissionException extends MMSException { public PermissionException(HttpStatus status, Object body) { super(status, body); } diff --git a/settings.gradle b/settings.gradle index 1250aaeba..4e94a0409 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ -rootProject.name = 'sdvc' +rootProject.name = 'mms' FileTree buildFiles = fileTree(rootDir) { List excludes = gradle.startParameter.projectProperties.get("excludeProjects")?.split(",") diff --git a/twc/src/main/java/org/openmbee/mms/twc/exceptions/TwcConfigurationException.java b/twc/src/main/java/org/openmbee/mms/twc/exceptions/TwcConfigurationException.java index 22485543d..f50e90532 100644 --- a/twc/src/main/java/org/openmbee/mms/twc/exceptions/TwcConfigurationException.java +++ b/twc/src/main/java/org/openmbee/mms/twc/exceptions/TwcConfigurationException.java @@ -1,9 +1,9 @@ package org.openmbee.mms.twc.exceptions; -import org.openmbee.mms.core.exceptions.SdvcException; +import org.openmbee.mms.core.exceptions.MMSException; import org.springframework.http.HttpStatus; -public class TwcConfigurationException extends SdvcException { +public class TwcConfigurationException extends MMSException { public TwcConfigurationException(HttpStatus code, Object messageObject) { super(code, messageObject); }