diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index a71633129..42b4699b2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.model.imports; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -28,6 +29,9 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import java.util.ArrayList; +import java.util.List; + @Getter @Setter @NoArgsConstructor @@ -38,6 +42,16 @@ public class PendingImport { private PendingImportObject location; private PendingImportObject study; private PendingImportObject observationUnit; - private PendingImportObject observation; + private List> observations = new ArrayList<>(); + + @JsonIgnore + public PendingImportObject getObservation() { + return observations.get(0); + } + + public void setObservation(PendingImportObject observation) { + observations.clear(); + observations.add(observation); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 85c21fb61..4042b5cc2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -24,6 +24,7 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -283,6 +284,16 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } + // TODO: Fill out with rest of data for saving to BRAPI + public BrAPIObservation constructBrAPIObservation(String value, String variableName) { + BrAPIObservation observation = new BrAPIObservation(); + + observation.setValue(value); + observation.setObservationVariableName(variableName); + + return observation; + } + private List getBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { List refs = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 753bd310e..b0adb1d9e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -22,6 +22,7 @@ import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -37,6 +38,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -107,6 +109,9 @@ public class ExperimentProcessor implements Processor { // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; + + private Map> observationByHash = new HashMap<>(); + // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -148,6 +153,8 @@ public void getExistingBrapiData(List importRows, Program program) this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); // All of the Observation Units will be new. None will be preexisting. this.observationUnitByNameNoScope = new HashMap<>(); + // TODO: populate existing observations, assume all new currently + // key and removing key this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); } @@ -214,7 +221,7 @@ public Map process( } // add "New" pending data to the BrapiData objects - getNewBrapiData(importRows, program, user, commit); + getNewBrapiData(importRows, phenotypeCols, program, user, commit); // For each import row for (int i = 0; i < importRows.size(); i++) { @@ -225,6 +232,16 @@ public Map process( mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + List> observations = mappedImportRow.getObservations(); + + // if value was blank won't be entry in map for this observation + PendingImportObject observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))); + observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + PendingImportObject germplasmPIO = getGidPOI(importRow); mappedImportRow.setGermplasm( germplasmPIO ); @@ -244,7 +261,7 @@ public Map process( validationErrors = validateFields(importRows, validationErrors); - if (validationErrors.hasErrors() ){ + if (validationErrors.hasErrors()){ throw new ValidatorException(validationErrors); } @@ -252,7 +269,13 @@ public Map process( return getStatisticsMap(importRows); } - private void getNewBrapiData(List importRows, Program program, User user, boolean commit) { + + private String getVariableNameFromColumn(Column column) { + // TODO: timestamp stripping? + return column.name(); + } + + private void getNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, boolean commit) { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -268,7 +291,8 @@ private void getNewBrapiData(List importRows, Program program, User } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - for (BrAPIImport row : importRows) { + for (int i=0; i trialPIO = createTrialPIO(program, user, commit, importRow, expNextVal); @@ -293,6 +317,11 @@ private void getNewBrapiData(List importRows, Program program, User PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow); String key = createObservationUnitKey(importRow); this.observationUnitByNameNoScope.put(key, obsUnitPIO); + + for (Column column : phenotypeCols) { + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i)); + this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + } } } @@ -301,6 +330,19 @@ private String createObservationUnitKey(ExperimentObservation importRow) { return key; } + private String getImportObservationHash(ExperimentObservation importRow, String variableName) { + // TODO: handle timestamps once we support them + return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); + } + + //TODO: Add timestamp parameter once we support them + private String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(studyName); + return DigestUtils.sha256Hex(concat); + } + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { HashSet uniqueStudyAndObsUnit = new HashSet<>(); for (int i = 0; i < importRows.size(); i++) { @@ -405,6 +447,7 @@ private Map getStatisticsMap(List HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's HashSet gidCounter = new HashSet<>(); // set of unique GID's + for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. @@ -412,6 +455,14 @@ private Map getStatisticsMap(List addIfNotNull(obsUnitsIDCounter, createObservationUnitKey( importRow )); addIfNotNull(gidCounter, importRow.getGid()); } + + int numNewObservations = Math.toIntExact( + observationByHash.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject().getValue())) + .count() + ); + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -421,11 +472,15 @@ private Map getStatisticsMap(List ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() .newObjectCount(gidCounter.size()) .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); return Map.of( "Environments", environmentStats, "Observation_Units", obdUnitStats, - "GIDs", gidStats + "GIDs", gidStats, + "Observations", observationStats ); } @@ -470,6 +525,19 @@ private PendingImportObject createObsUnitPIO(Program progr return pio; } + + private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value) { + PendingImportObject pio = null; + if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { + pio = observationByHash.get(getImportObservationHash(importRow, variableName)); + } + else { + BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + } + return pio; + } + private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 8dc9e147d..83cf4d2ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -167,7 +167,6 @@ public Map process(List importRows mappedImportRow.setObservation(observationByHash.get(hash)); mappedBrAPIImport.put(i, mappedImportRow); } - } }