From 94c0080fdada563b4272de032751e14c3e120d7f Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Fri, 7 Oct 2022 10:46:22 -0400 Subject: [PATCH 01/10] Work in progress --- .../processors/ExperimentProcessor.java | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 786036c18..493d14250 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -22,6 +22,7 @@ import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -36,6 +37,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -104,6 +106,9 @@ public class ExperimentProcessor implements Processor { // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; + + private Map> observationByHash = new HashMap<>(); + // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -145,6 +150,8 @@ public void getExistingBrapiData(List importRows, Program program) this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); // All of the Observation Units will be new. None will be preexisting. this.observationUnitByNameNoScope = new HashMap<>(); + // TODO: populate existing observations, assume all new currently + // key and removing key this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); } @@ -211,7 +218,7 @@ public Map process( } // add "New" pending data to the BrapiData objects - getNewBrapiData(importRows, program, commit); + getNewBrapiData(importRows, List> phenotypeCols, program, commit); // For each import row for (int i = 0; i < importRows.size(); i++) { @@ -222,6 +229,12 @@ public Map process( mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + mappedImportRow.setObservation(); + } + PendingImportObject germplasmPIO = getGidPOI(importRow); mappedImportRow.setGermplasm( germplasmPIO ); @@ -249,7 +262,7 @@ public Map process( return getStatisticsMap(importRows); } - private void getNewBrapiData(List importRows, Program program, boolean commit) { + private void getNewBrapiData(List importRows, List> phenotypeCols, Program program, boolean commit) { String obsUnitSequenceName = program.getObsUnitSequence(); if (obsUnitSequenceName == null) { @@ -284,14 +297,31 @@ private void getNewBrapiData(List importRows, Program program, bool PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); String key = createObservationUnitKey(importRow); this.observationUnitByNameNoScope.put(key, obsUnitPIO); + + for (Column column : phenotypeCols) { + PendingImportObject obsPIO = + } } } - private String createObservationUnitKey(ExperimentObservation importRow) { + private static String createObservationUnitKey(ExperimentObservation importRow) { String key = importRow.getEnv() + importRow.getExpUnitId(); return key; } + private static String getImportObservationHash(ExperimentObservation importRow) { + // TODO: handle timestamps once we support them + return getObservationHash(createObservationUnitKey(importRow), ,importRow.getEnv()); + } + + //TODO: Add timestamp parameter once we support them + private static String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(studyName); + return DigestUtils.sha256Hex(concat); + } + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { HashSet uniqueStudyAndObsUnit = new HashSet<>(); for (int i = 0; i < importRows.size(); i++) { @@ -461,6 +491,27 @@ private PendingImportObject createObsUnitPIO(Program progr return pio; } + private PendingImportObject createObservationPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { + PendingImportObject pio = null; + if ( this.observationByHash.containsKey( createObservationUnitKey( importRow ) ) ) { + pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; + } + else { + String germplasmName = ""; + if( this.existingGermplasmByGID.get( importRow.getGid() ) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()).getBrAPIObject().getGermplasmName(); + } + PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + } + return pio; + } + private PendingImportObject createStudyPIO(Program program, boolean commit, String expSeqenceValue, ExperimentObservation importRow) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { From c29128b506217b6ba548c1b5248a24a0f53247db Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Mon, 10 Oct 2022 15:00:02 -0400 Subject: [PATCH 02/10] Save before update --- .../processors/ExperimentProcessor.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 493d14250..379ae5f47 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -218,7 +218,7 @@ public Map process( } // add "New" pending data to the BrapiData objects - getNewBrapiData(importRows, List> phenotypeCols, program, commit); + getNewBrapiData(importRows, phenotypeCols, program, commit); // For each import row for (int i = 0; i < importRows.size(); i++) { @@ -232,7 +232,7 @@ public Map process( // loop over phenotype column observation data for current row for (Column column : phenotypeCols) { - mappedImportRow.setObservation(); + mappedImportRow.setObservation(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); } PendingImportObject germplasmPIO = getGidPOI(importRow); @@ -254,7 +254,7 @@ public Map process( validationErrors = validateFields(importRows, validationErrors); - if (validationErrors.hasErrors() ){ + if (validationErrors.hasErrors()){ throw new ValidatorException(validationErrors); } @@ -262,6 +262,11 @@ public Map process( return getStatisticsMap(importRows); } + private String getVariableNameFromColumn(Column column) { + // TOO: timestamp stripping? + return column.name(); + } + private void getNewBrapiData(List importRows, List> phenotypeCols, Program program, boolean commit) { String obsUnitSequenceName = program.getObsUnitSequence(); @@ -299,7 +304,7 @@ private void getNewBrapiData(List importRows, List> pheno this.observationUnitByNameNoScope.put(key, obsUnitPIO); for (Column column : phenotypeCols) { - PendingImportObject obsPIO = + PendingImportObject obsPIO = createObservationPIO(program, commit, obsUnitNextVal, importRow); } } } @@ -309,9 +314,9 @@ private static String createObservationUnitKey(ExperimentObservation importRow) return key; } - private static String getImportObservationHash(ExperimentObservation importRow) { + private static String getImportObservationHash(ExperimentObservation importRow, String variableName) { // TODO: handle timestamps once we support them - return getObservationHash(createObservationUnitKey(importRow), ,importRow.getEnv()); + return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); } //TODO: Add timestamp parameter once we support them @@ -491,10 +496,10 @@ private PendingImportObject createObsUnitPIO(Program progr return pio; } - private PendingImportObject createObservationPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { + private PendingImportObject createObservationPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow, String variableName) { PendingImportObject pio = null; - if ( this.observationByHash.containsKey( createObservationUnitKey( importRow ) ) ) { - pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; + if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { + pio = observationByHash.get(getImportObservationHash(importRow, variableName)); } else { String germplasmName = ""; @@ -507,6 +512,8 @@ private PendingImportObject createObservationPIO(Program progr UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); } return pio; From 4c200690cdbb5ea087b64698bcfbc624ae3fad7b Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 25 Oct 2022 13:26:53 -0400 Subject: [PATCH 03/10] Minimal observation saving --- .../importer/model/imports/PendingImport.java | 17 ++++++++++- .../ExperimentObservation.java | 9 ++++++ .../processors/ExperimentProcessor.java | 28 ++++++++----------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index a71633129..003c557f2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.model.imports; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -28,6 +29,9 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import java.util.ArrayList; +import java.util.List; + @Getter @Setter @NoArgsConstructor @@ -38,6 +42,17 @@ public class PendingImport { private PendingImportObject location; private PendingImportObject study; private PendingImportObject observationUnit; - private PendingImportObject observation; + private List> observations = new ArrayList<>(); + + + @JsonIgnore + public PendingImportObject getObservation() { + return observations.get(0); + } + + public void setObservation(PendingImportObject observation) { + observations.clear(); + observations.add(observation); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index cb64a5a9a..3ef60d54e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -24,6 +24,7 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -262,6 +263,14 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } + public BrAPIObservation constructBrAPIObservation(String value) { + BrAPIObservation observation = new BrAPIObservation(); + + observation.setValue(value); + + return observation; + } + private List getBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { List refs = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 379ae5f47..78bbbb087 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -232,7 +232,10 @@ public Map process( // loop over phenotype column observation data for current row for (Column column : phenotypeCols) { - mappedImportRow.setObservation(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + List> observations = mappedImportRow.getObservations(); + observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + + //mappedImportRow.setObservations(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); } PendingImportObject germplasmPIO = getGidPOI(importRow); @@ -287,7 +290,8 @@ private void getNewBrapiData(List importRows, List> pheno expSeqValue = dsl.nextval(expUnitSequenceName.toLowerCase()).toString(); } - for (BrAPIImport row : importRows) { + for (int i=0; i trialPIO = createTrialPIO(program, commit, importRow, expSeqValue); @@ -304,7 +308,8 @@ private void getNewBrapiData(List importRows, List> pheno this.observationUnitByNameNoScope.put(key, obsUnitPIO); for (Column column : phenotypeCols) { - PendingImportObject obsPIO = createObservationPIO(program, commit, obsUnitNextVal, importRow); + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i)); + this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); } } } @@ -496,25 +501,14 @@ private PendingImportObject createObsUnitPIO(Program progr return pio; } - private PendingImportObject createObservationPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow, String variableName) { + private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value) { PendingImportObject pio = null; if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { pio = observationByHash.get(getImportObservationHash(importRow, variableName)); } else { - String germplasmName = ""; - if( this.existingGermplasmByGID.get( importRow.getGid() ) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()).getBrAPIObject().getGermplasmName(); - } - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); - UUID trialID = trialPIO.getId(); - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); - - - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + BrAPIObservation newObservation = importRow.constructBrAPIObservation(value); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); } return pio; } From a08418859534683347e194ce23c0bda2a2b59e16 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 25 Oct 2022 15:09:24 -0400 Subject: [PATCH 04/10] Add comment for filling out brapi data --- .../imports/experimentObservation/ExperimentObservation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 3ef60d54e..11820f05c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -263,6 +263,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } + // TODO: Fill out with rest of data for saving to BRAPI public BrAPIObservation constructBrAPIObservation(String value) { BrAPIObservation observation = new BrAPIObservation(); From c9e26628508fa48bfdc6949b9c3a6f9b4e0931d2 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 25 Oct 2022 15:27:04 -0400 Subject: [PATCH 05/10] Some cleanup --- .../importer/services/processors/ExperimentProcessor.java | 4 +--- .../importer/services/processors/ObservationProcessor.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 78bbbb087..434655e01 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -234,8 +234,6 @@ public Map process( for (Column column : phenotypeCols) { List> observations = mappedImportRow.getObservations(); observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - - //mappedImportRow.setObservations(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); } PendingImportObject germplasmPIO = getGidPOI(importRow); @@ -266,7 +264,7 @@ public Map process( } private String getVariableNameFromColumn(Column column) { - // TOO: timestamp stripping? + // TODO: timestamp stripping? return column.name(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 8dc9e147d..83cf4d2ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -167,7 +167,6 @@ public Map process(List importRows mappedImportRow.setObservation(observationByHash.get(hash)); mappedBrAPIImport.put(i, mappedImportRow); } - } } From 9768f133a257f16801cb07067b461c76747bcab3 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 25 Oct 2022 15:29:30 -0400 Subject: [PATCH 06/10] Remove space --- .../brapps/importer/model/imports/PendingImport.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index 003c557f2..42b4699b2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -44,7 +44,6 @@ public class PendingImport { private PendingImportObject observationUnit; private List> observations = new ArrayList<>(); - @JsonIgnore public PendingImportObject getObservation() { return observations.get(0); From aef807efb70d8bcec662c12351aec02e4158e9b5 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Tue, 25 Oct 2022 17:17:19 -0400 Subject: [PATCH 07/10] Added statistics and blank value check --- .../processors/ExperimentProcessor.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d4549f3f5..8780b0865 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -236,7 +236,12 @@ public Map process( // loop over phenotype column observation data for current row for (Column column : phenotypeCols) { List> observations = mappedImportRow.getObservations(); - observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + + // if value was blank won't be entry in map for this observation + PendingImportObject observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))); + if (observation != null) { + observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } } PendingImportObject germplasmPIO = getGidPOI(importRow); @@ -315,8 +320,10 @@ private void getNewBrapiData(List importRows, List> pheno this.observationUnitByNameNoScope.put(key, obsUnitPIO); for (Column column : phenotypeCols) { - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i)); - this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + if (!StringUtils.isBlank(column.getString(i))) { + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i)); + this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + } } } } @@ -443,6 +450,7 @@ private Map getStatisticsMap(List HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's HashSet gidCounter = new HashSet<>(); // set of unique GID's + for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. @@ -450,6 +458,7 @@ private Map getStatisticsMap(List addIfNotNull(obsUnitsIDCounter, createObservationUnitKey( importRow )); addIfNotNull(gidCounter, importRow.getGid()); } + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -459,11 +468,15 @@ private Map getStatisticsMap(List ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() .newObjectCount(gidCounter.size()) .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(ProcessorData.getNumNewObjects(observationByHash)) + .build(); return Map.of( "Environments", environmentStats, "Observation_Units", obdUnitStats, - "GIDs", gidStats + "GIDs", gidStats, + "Observations", observationStats ); } From 43df9d0a3b455a344a5a0df7e43ce2b5b33b3820 Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 3 Nov 2022 14:16:43 -0400 Subject: [PATCH 08/10] Added observation variable name to observation construction --- .../imports/experimentObservation/ExperimentObservation.java | 3 ++- .../importer/services/processors/ExperimentProcessor.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 1e831b4a4..f39c5db73 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -269,10 +269,11 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } // TODO: Fill out with rest of data for saving to BRAPI - public BrAPIObservation constructBrAPIObservation(String value) { + public BrAPIObservation constructBrAPIObservation(String value, String variableName) { BrAPIObservation observation = new BrAPIObservation(); observation.setValue(value); + observation.setObservationVariableName(variableName); return observation; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 8780b0865..7e63434b2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -528,7 +528,7 @@ private PendingImportObject createObservationPIO(ExperimentObs pio = observationByHash.get(getImportObservationHash(importRow, variableName)); } else { - BrAPIObservation newObservation = importRow.constructBrAPIObservation(value); + BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); } return pio; From f5c7f1965070a37340035763ec0a490ab173a9ae Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Thu, 3 Nov 2022 15:09:22 -0400 Subject: [PATCH 09/10] Allow blank observation values but don't count in statistics --- .../processors/ExperimentProcessor.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7e63434b2..c497b8b38 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -239,9 +239,7 @@ public Map process( // if value was blank won't be entry in map for this observation PendingImportObject observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))); - if (observation != null) { - observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } + observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); } PendingImportObject germplasmPIO = getGidPOI(importRow); @@ -320,10 +318,8 @@ private void getNewBrapiData(List importRows, List> pheno this.observationUnitByNameNoScope.put(key, obsUnitPIO); for (Column column : phenotypeCols) { - if (!StringUtils.isBlank(column.getString(i))) { - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i)); - this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); - } + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i)); + this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); } } } @@ -459,6 +455,13 @@ private Map getStatisticsMap(List addIfNotNull(gidCounter, importRow.getGid()); } + int numNewObservations = Math.toIntExact( + observationByHash.values().stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject().getValue())) + .count() + ); + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -469,7 +472,7 @@ private Map getStatisticsMap(List .newObjectCount(gidCounter.size()) .build(); ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(ProcessorData.getNumNewObjects(observationByHash)) + .newObjectCount(numNewObservations) .build(); return Map.of( From 113be0dd792d1500cebcd01a42e8efdecdcc7dfa Mon Sep 17 00:00:00 2001 From: Nick Palladino Date: Mon, 7 Nov 2022 11:20:47 -0500 Subject: [PATCH 10/10] Code review updates --- .../importer/services/processors/ExperimentProcessor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 11832ca05..b0adb1d9e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -325,18 +325,18 @@ private void getNewBrapiData(List importRows, List> pheno } } - private static String createObservationUnitKey(ExperimentObservation importRow) { + private String createObservationUnitKey(ExperimentObservation importRow) { String key = importRow.getEnv() + importRow.getExpUnitId(); return key; } - private static String getImportObservationHash(ExperimentObservation importRow, String variableName) { + private String getImportObservationHash(ExperimentObservation importRow, String variableName) { // TODO: handle timestamps once we support them return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); } //TODO: Add timestamp parameter once we support them - private static String getObservationHash(String observationUnitName, String variableName, String studyName) { + private String getObservationHash(String observationUnitName, String variableName, String studyName) { String concat = DigestUtils.sha256Hex(observationUnitName) + DigestUtils.sha256Hex(variableName) + DigestUtils.sha256Hex(studyName);