BI-2639 - List referencing existing germplasm - very slow and impacting jobs timer

src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java

@@ -155,7 +155,19 @@ public static String constructGermplasmListName(String listName, Program program
         return String.format("%s [%s-germplasm]", listName, program.getKey());
     }
 
-    public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) {
+    /**
+     * Will mutate synonym and pedigree fields if changed and meet change criteria
+     *
+     * @param germplasm germplasm object
+     * @param program program
+     * @param listId list id
+     * @param commit flag indicating if commit changes should be made
+     * @param updatePedigree flag indicating if pedigree should be updated
+     * @return mutated indicator
+     */
+    public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) {
+
+        boolean mutated = false;
 
         if (updatePedigree) {
             if (!StringUtils.isBlank(getFemaleParentAccessionNumber())) {
@@ -170,6 +182,7 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID
             if (!StringUtils.isBlank(getMaleParentEntryNo())) {
                 germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO, getMaleParentEntryNo());
             }
+            mutated = true;
         }
 
         // Append synonyms to germplasm that don't already exist
@@ -181,6 +194,7 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID
                 brapiSynonym.setSynonym(synonym);
                 if (!existingSynonyms.contains(brapiSynonym)) {
                     germplasm.addSynonymsItem(brapiSynonym);
+                    mutated = true;
                 }
             }
         }
@@ -193,6 +207,8 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID
         if (commit) {
             setUpdateCommitFields(germplasm, program.getKey());
         }
+
+        return mutated;
     }
 
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java

@@ -87,6 +87,8 @@ public class GermplasmProcessor implements Processor {
     List<List<BrAPIGermplasm>> postOrder = new ArrayList<>();
     BrAPIListNewRequest importList = new BrAPIListNewRequest();
 
+    private int numNewPedigreeConnections = 0;
+
     public static String missingGIDsMsg = "The following GIDs were not found in the database: %s";
     public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s";
     public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s";
@@ -332,7 +334,7 @@ public Map<String, ImportPreviewStatistics> process(ImportUpload upload, List<Br
         createPostOrder();
 
         // Construct our response object
-        return getStatisticsMap(importRows);
+        return getStatisticsMap();
     }
 
     private void processNewGermplasm(Germplasm germplasm, ValidationErrors validationErrors, Map<String, ProgramBreedingMethodEntity> breedingMethods,
@@ -359,6 +361,10 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio
 
         validatePedigree(germplasm, i + 2, validationErrors);
 
+        if (germplasm.pedigreeExists()) {
+            numNewPedigreeConnections++;
+        }
+
         BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId);
 
         newGermplasmList.add(newGermplasm);
@@ -383,6 +389,9 @@ private Germplasm removeBreedingMethodBlanks(Germplasm germplasm) {
     private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors validationErrors, List<BrAPIImport> importRows, Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int rowIndex) {
         BrAPIGermplasm existingGermplasm;
         String gid = germplasm.getAccessionNumber();
+        boolean mutated = false;
+        boolean updatePedigree = false;
+
         if (germplasmByAccessionNumber.containsKey(gid)) {
             existingGermplasm = germplasmByAccessionNumber.get(gid).getBrAPIObject();
             // Serialize and deserialize to deep copy
@@ -408,17 +417,26 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v
             }
         }
 
-        if(germplasm.pedigreeExists()) {
+        // if no existing pedigree and file has pedigree then validate and update
+        if(germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) {
             validatePedigree(germplasm, rowIndex + 2, validationErrors);
+            updatePedigree = true;
         }
 
-        germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, true);
-
-        updatedGermplasmList.add(existingGermplasm);
-        mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm));
-        importList.addDataItem(existingGermplasm.getGermplasmName());
+        mutated = germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, updatePedigree);
 
+        if (mutated) {
+            updatedGermplasmList.add(existingGermplasm);
+            mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm));
+            if (updatePedigree) {
+                numNewPedigreeConnections++;
+            }
+        } else {
+            mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm));
+        }
 
+        // add to list regardless of mutated or not
+        importList.addDataItem(existingGermplasm.getGermplasmName());
         return true;
     }
 
@@ -521,20 +539,17 @@ private boolean canUpdatePedigreeNoEqualsCheck(BrAPIGermplasm existingGermplasm,
                 germplasm.pedigreeExists();
     }
 
-    private Map<String, ImportPreviewStatistics> getStatisticsMap(List<BrAPIImport> importRows) {
+    private Map<String, ImportPreviewStatistics> getStatisticsMap() {
 
         ImportPreviewStatistics germplasmStats = ImportPreviewStatistics.builder()
                 .newObjectCount(newGermplasmList.size())
                 .ignoredObjectCount(germplasmByAccessionNumber.size())
                 .build();
 
-        //Modified logic here to check for female parent accession number or entry no, removed check for male due to assumption that shouldn't have only male parent
-        int newObjectCount = newGermplasmList.stream().filter(newGermplasm -> newGermplasm != null).collect(Collectors.toList()).size();
+        // TODO: numNewPedigreeConnections is global modified in existing and new flows, refactor at some point
         ImportPreviewStatistics pedigreeConnectStats = ImportPreviewStatistics.builder()
-                .newObjectCount(importRows.stream().filter(germplasmImport ->
-                        germplasmImport.getGermplasm() != null &&
-                                (germplasmImport.getGermplasm().getFemaleParentAccessionNumber() != null || germplasmImport.getGermplasm().getFemaleParentEntryNo() != null)
-                ).collect(Collectors.toList()).size()).build();
+                .newObjectCount(numNewPedigreeConnections)
+                .build();
 
         return Map.of(
                 "Germplasm", germplasmStats,
@@ -631,7 +646,8 @@ public void postBrapiData(Map<Integer, PendingImport> mappedBrAPIImport, Program
         }
 
         // Create list
-        if (!newGermplasmList.isEmpty() || !updatedGermplasmList.isEmpty()) {
+        // create & update flows both unconditionally add germplasm names to importList so use that for check
+        if (!importList.getData().isEmpty()) {
             try {
                 // Create germplasm list
                 brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload);