BI-1612 - Experiment Import Phenotype Data Persistence

src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java

@@ -17,6 +17,7 @@
 
 package org.breedinginsight.brapps.importer.model.imports;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -28,6 +29,9 @@
 import org.brapi.v2.model.pheno.BrAPIObservationUnit;
 import org.breedinginsight.brapps.importer.model.response.PendingImportObject;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @Getter
 @Setter
 @NoArgsConstructor
@@ -38,6 +42,16 @@ public class PendingImport {
     private PendingImportObject<BrAPILocation> location;
     private PendingImportObject<BrAPIStudy> study;
     private PendingImportObject<BrAPIObservationUnit> observationUnit;
-    private PendingImportObject<BrAPIObservation> observation;
+    private List<PendingImportObject<BrAPIObservation>> observations = new ArrayList<>();
+
+    @JsonIgnore
+    public PendingImportObject<BrAPIObservation> getObservation() {
+        return observations.get(0);
+    }
+
+    public void setObservation(PendingImportObject<BrAPIObservation> observation) {
+        observations.clear();
+        observations.add(observation);
+    }
 
 }

src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java

@@ -24,6 +24,7 @@
 import org.brapi.v2.model.core.*;
 import org.brapi.v2.model.pheno.*;
 import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
+import org.breedinginsight.brapps.importer.model.base.Observation;
 import org.breedinginsight.brapps.importer.model.config.*;
 import org.breedinginsight.brapps.importer.model.imports.BrAPIImport;
 import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
@@ -283,6 +284,16 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(
         return observationUnit;
     }
 
+    // TODO: Fill out with rest of data for saving to BRAPI
+    public BrAPIObservation constructBrAPIObservation(String value, String variableName) {
+        BrAPIObservation observation = new BrAPIObservation();
+
+        observation.setValue(value);
+        observation.setObservationVariableName(variableName);
+
+        return observation;
+    }
+
     private List<BrAPIExternalReference> getBrAPIExternalReferences(
             Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) {
         List<BrAPIExternalReference> refs = new ArrayList<>();

src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java

@@ -22,6 +22,7 @@
 import io.micronaut.http.exceptions.HttpStatusException;
 import io.micronaut.http.server.exceptions.InternalServerException;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.brapi.client.v2.model.exceptions.ApiException;
 import org.brapi.v2.model.BrAPIExternalReference;
@@ -37,6 +38,7 @@
 import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO;
 import org.breedinginsight.brapps.importer.daos.*;
 import org.breedinginsight.brapps.importer.model.ImportUpload;
+import org.breedinginsight.brapps.importer.model.base.Observation;
 import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation;
 import org.breedinginsight.brapps.importer.model.imports.BrAPIImport;
 import org.breedinginsight.brapps.importer.model.imports.PendingImport;
@@ -107,6 +109,9 @@ public class ExperimentProcessor implements Processor {
     //  It is assumed that there are no preexisting Observation Units for the given environment (so this will not be
     // initialized by getExistingBrapiData() )
     private Map<String, PendingImportObject<BrAPIObservationUnit>> observationUnitByNameNoScope = null;
+
+    private Map<String, PendingImportObject<BrAPIObservation>> observationByHash = new HashMap<>();
+
     // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method
     private Map<String, PendingImportObject<BrAPIGermplasm>> existingGermplasmByGID = null;
 
@@ -148,6 +153,8 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)
         this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows );
         // All of the Observation Units will be new.  None will be preexisting.
         this.observationUnitByNameNoScope =  new HashMap<>();
+        // TODO: populate existing observations, assume all new currently
+        // key and removing key
         this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows );
     }
 
@@ -214,7 +221,7 @@ public Map<String, ImportPreviewStatistics> process(
         }
 
         // add "New" pending data to the BrapiData objects
-        getNewBrapiData(importRows, program, user, commit);
+        getNewBrapiData(importRows, phenotypeCols, program, user, commit);
 
         // For each import row
         for (int i = 0; i < importRows.size(); i++) {
@@ -225,6 +232,16 @@ public Map<String, ImportPreviewStatistics> process(
             mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) );
             mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) );
             mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) );
+
+            // loop over phenotype column observation data for current row
+            for (Column<?> column : phenotypeCols) {
+                List<PendingImportObject<BrAPIObservation>> observations = mappedImportRow.getObservations();
+
+                // if value was blank won't be entry in map for this observation
+                PendingImportObject<BrAPIObservation> observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)));
+                observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))));
+            }
+
             PendingImportObject<BrAPIGermplasm> germplasmPIO = getGidPOI(importRow);
             mappedImportRow.setGermplasm( germplasmPIO );
 
@@ -244,15 +261,21 @@ public Map<String, ImportPreviewStatistics> process(
 
         validationErrors = validateFields(importRows, validationErrors);
 
-        if (validationErrors.hasErrors() ){
+        if (validationErrors.hasErrors()){
             throw new ValidatorException(validationErrors);
         }
 
         // Construct our response object
         return getStatisticsMap(importRows);
     }
 
-    private void getNewBrapiData(List<BrAPIImport> importRows, Program program, User user, boolean commit) {
+
+    private String getVariableNameFromColumn(Column<?> column) {
+        // TODO: timestamp stripping?
+        return column.name();
+    }
+
+    private void getNewBrapiData(List<BrAPIImport> importRows, List<Column<?>> phenotypeCols, Program program, User user, boolean commit) {
 
         String expSequenceName = program.getExpSequence();
         if (expSequenceName == null) {
@@ -268,7 +291,8 @@ private void getNewBrapiData(List<BrAPIImport> importRows, Program program, User
         }
         Supplier<BigInteger> envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase());
 
-        for (BrAPIImport row : importRows) {
+        for (int i=0; i<importRows.size(); i++) {
+            BrAPIImport row = importRows.get(i);
             ExperimentObservation importRow = (ExperimentObservation) row;
 
             PendingImportObject<BrAPITrial> trialPIO = createTrialPIO(program, user, commit, importRow, expNextVal);
@@ -293,6 +317,11 @@ private void getNewBrapiData(List<BrAPIImport> importRows, Program program, User
             PendingImportObject<BrAPIObservationUnit> obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow);
             String key = createObservationUnitKey(importRow);
             this.observationUnitByNameNoScope.put(key, obsUnitPIO);
+
+            for (Column<?> column : phenotypeCols) {
+                PendingImportObject<BrAPIObservation> obsPIO = createObservationPIO(importRow, column.name(), column.getString(i));
+                this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO);
+            }
         }
     }
 
@@ -301,6 +330,19 @@ private String createObservationUnitKey(ExperimentObservation importRow) {
         return key;
     }
 
+    private String getImportObservationHash(ExperimentObservation importRow, String variableName) {
+        // TODO: handle timestamps once we support them
+        return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv());
+    }
+
+    //TODO: Add timestamp parameter once we support them
+    private String getObservationHash(String observationUnitName, String variableName, String studyName) {
+        String concat = DigestUtils.sha256Hex(observationUnitName) +
+                DigestUtils.sha256Hex(variableName) +
+                DigestUtils.sha256Hex(studyName);
+        return DigestUtils.sha256Hex(concat);
+    }
+
     private ValidationErrors validateFields(List<BrAPIImport> importRows, ValidationErrors validationErrors) {
         HashSet<String> uniqueStudyAndObsUnit = new HashSet<>();
         for (int i = 0; i < importRows.size(); i++) {
@@ -405,13 +447,22 @@ private Map<String, ImportPreviewStatistics> getStatisticsMap(List<BrAPIImport>
         HashSet<String> environmentNameCounter = new HashSet<>(); // set of unique environment names
         HashSet<String> obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's
         HashSet<String> gidCounter = new HashSet<>(); // set of unique GID's
+
         for (BrAPIImport row : importRows) {
             ExperimentObservation importRow = (ExperimentObservation) row;
             // Collect date for stats.
             addIfNotNull(environmentNameCounter, importRow.getEnv());
             addIfNotNull(obsUnitsIDCounter, createObservationUnitKey( importRow ));
             addIfNotNull(gidCounter, importRow.getGid());
         }
+
+        int numNewObservations = Math.toIntExact(
+                observationByHash.values().stream()
+                .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW &&
+                        !StringUtils.isBlank(preview.getBrAPIObject().getValue()))
+                .count()
+        );
+
         ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder()
                 .newObjectCount(environmentNameCounter.size())
                 .build();
@@ -421,11 +472,15 @@ private Map<String, ImportPreviewStatistics> getStatisticsMap(List<BrAPIImport>
         ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder()
                 .newObjectCount(gidCounter.size())
                 .build();
+        ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder()
+                .newObjectCount(numNewObservations)
+                .build();
 
         return Map.of(
                 "Environments",         environmentStats,
                 "Observation_Units",    obdUnitStats,
-                "GIDs",                 gidStats
+                "GIDs",                 gidStats,
+                "Observations",         observationStats
         );
     }
 
@@ -470,6 +525,19 @@ private PendingImportObject<BrAPIObservationUnit> createObsUnitPIO(Program progr
         return pio;
     }
 
+
+    private PendingImportObject<BrAPIObservation> createObservationPIO(ExperimentObservation importRow, String variableName, String value) {
+        PendingImportObject<BrAPIObservation> pio = null;
+        if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) {
+            pio = observationByHash.get(getImportObservationHash(importRow, variableName));
+        }
+        else {
+            BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName);
+            pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation);
+        }
+        return pio;
+    }
+
     private PendingImportObject<BrAPIStudy> createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier<BigInteger> envNextVal) {
         PendingImportObject<BrAPIStudy> pio = null;
         if( studyByNameNoScope.containsKey( importRow.getEnv()) ) {

src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java

@@ -167,7 +167,6 @@ public Map<String, ImportPreviewStatistics> process(List<BrAPIImport> importRows
                     mappedImportRow.setObservation(observationByHash.get(hash));
                     mappedBrAPIImport.put(i, mappedImportRow);
                 }
-
             }
         }