From 44a394d9117aa316caee9f786422be3f1d26b792 Mon Sep 17 00:00:00 2001
From: "Lam, Doris T (319E)" <Doris.T.Lam@jpl.nasa.gov>
Date: Wed, 2 Dec 2020 12:42:16 -0800
Subject: [PATCH 1/2] chunk elastic gets

---
 .../mms/crud/services/DefaultNodeService.java |  4 +-
 .../mms/elastic/BaseElasticDAOImpl.java       | 37 +++++++++++++------
 .../resources/application-test.properties     |  1 +
 .../resources/application.properties.example  |  1 +
 4 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java
index 81ec448fc..a823df6ca 100644
--- a/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java
+++ b/crud/src/main/java/org/openmbee/mms/crud/services/DefaultNodeService.java
@@ -168,9 +168,9 @@ protected void commitChanges(NodeChangeInfo info) {
                 commit.setDocId(cmjs.getId());
                 commit.setTimestamp(now);
                 commit.setComment(cmjs.getComment());
-
-                this.commitIndex.index(cmjs);
+                
                 this.commitRepository.save(commit);
+                this.commitIndex.index(cmjs);
                 this.nodeRepository.getTransactionManager().commit(status);
             } catch (Exception e) {
                 logger.error("commitChanges error: ", e);
diff --git a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java
index cc870b4d8..b33a70d26 100644
--- a/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java
+++ b/elastic/src/main/java/org/openmbee/mms/elastic/BaseElasticDAOImpl.java
@@ -46,6 +46,8 @@ public abstract class BaseElasticDAOImpl<E extends Map<String, Object>> {
     protected int resultLimit;
     @Value("${elasticsearch.limit.term}")
     protected int termLimit;
+    @Value("${elasticsearch.limit.get}")
+    protected int getLimit;
     protected static int readTimeout = 1000000000;
     protected RestHighLevelClient client;
     private static final RequestOptions REQUEST_OPTIONS;
@@ -121,28 +123,41 @@ public List<E> findAllById(String index, Set<String> docIds) {
             if (docIds.isEmpty()) {
                 return listOfResponses;
             }
+            int cur = 0;
             MultiGetRequest request = new MultiGetRequest();
             for (String eid : docIds) {
                 request.add(index, eid);
-            }
-            MultiGetResponse response = client.mget(request, REQUEST_OPTIONS);
-
-            for (MultiGetItemResponse res : response.getResponses()) {
-                GetResponse item = res.getResponse();
-                if (item != null && item.isExists()) {
-                    E ob = newInstance();
-                    ob.putAll(item.getSourceAsMap());
-                    listOfResponses.add(ob);
-                } else {
-                    continue;
+                cur++;
+                if (cur == getLimit) {
+                    getResponses(request, listOfResponses);
+                    cur = 0;
+                    request = new MultiGetRequest();
                 }
             }
+            if (cur > 0) {
+                getResponses(request, listOfResponses);
+            }
             return listOfResponses;
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
     }
 
+    private void getResponses(MultiGetRequest request, List<E> listOfResponses) throws IOException {
+        MultiGetResponse response = client.mget(request, REQUEST_OPTIONS);
+
+        for (MultiGetItemResponse res : response.getResponses()) {
+            GetResponse item = res.getResponse();
+            if (item != null && item.isExists()) {
+                E ob = newInstance();
+                ob.putAll(item.getSourceAsMap());
+                listOfResponses.add(ob);
+            } else {
+                continue;
+            }
+        }
+    }
+
     public void indexAll(String index, Collection<? extends BaseJson> jsons) {
         BulkProcessor bulkProcessor = getBulkProcessor(client);
         for (BaseJson json : jsons) {
diff --git a/example/src/main/resources/application-test.properties b/example/src/main/resources/application-test.properties
index 692a63551..2cb508f36 100644
--- a/example/src/main/resources/application-test.properties
+++ b/example/src/main/resources/application-test.properties
@@ -51,6 +51,7 @@ elasticsearch.limit.insert=80
 elasticsearch.limit.result=10000
 elasticsearch.limit.term=1000
 elasticsearch.limit.scrollTimeout=1000
+elasticsearch.limit.get=10000
 
 #Configuration for TWC
 #port is for REST interface
diff --git a/example/src/main/resources/application.properties.example b/example/src/main/resources/application.properties.example
index b0a3ee78c..6d0d11e24 100644
--- a/example/src/main/resources/application.properties.example
+++ b/example/src/main/resources/application.properties.example
@@ -49,6 +49,7 @@ elasticsearch.limit.insert=80
 elasticsearch.limit.result=10000
 elasticsearch.limit.term=1000
 elasticsearch.limit.scrollTimeout=1000
+elasticsearch.limit.get=100000
 
 #Configuration for TWC
 #port is for REST interface

From f1fde45f4e841edc58c6a50b0996434bb56546b6 Mon Sep 17 00:00:00 2001
From: "Lam, Doris T (319E)" <Doris.T.Lam@jpl.nasa.gov>
Date: Wed, 2 Dec 2020 13:35:53 -0800
Subject: [PATCH 2/2] repo order

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index a4ddf2f8b..7c02a7ca2 100644
--- a/build.gradle
+++ b/build.gradle
@@ -50,9 +50,9 @@ ext.milestoneBuild = !(snapshotBuild || releaseBuild)
 
 subprojects {
     repositories {
-        maven { url 'https://repo.spring.io/plugins-release' }
-        gradlePluginPortal()
         mavenCentral()
+        gradlePluginPortal()
+        maven { url 'https://repo.spring.io/plugins-release' }
     }
 
     plugins.withType(JavaPlugin) {