From 1595a201d90447495a0dab0fc0d93c76cd5db45c Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 13 Dec 2022 13:06:44 -0500 Subject: [PATCH 1/9] [BI-1665] Save each observation to the BrAPI backend --- .../constants/BrAPIAdditionalInfoFields.java | 1 + .../ExperimentObservation.java | 20 ++- .../processors/ExperimentProcessor.java | 131 +++++++++++++++--- .../java/org/breedinginsight/model/Trait.java | 3 +- 4 files changed, 132 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 3968eb7e0..1a633b9a1 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -37,4 +37,5 @@ public final class BrAPIAdditionalInfoFields { public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; public static final String ENVIRONMENT_NUMBER = "environmentNumber"; + public static final String STUDY_NAME = "studyName"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 4042b5cc2..d3e0540dc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -284,12 +284,24 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } - // TODO: Fill out with rest of data for saving to BRAPI - public BrAPIObservation constructBrAPIObservation(String value, String variableName) { + public BrAPIObservation constructBrAPIObservation( + String value, + String variableName, + String seasonDbId + ) { BrAPIObservation observation = new BrAPIObservation(); - - observation.setValue(value); + observation.setGermplasmDbId(getGid()); + observation.setGermplasmName(getGermplasmName()); + if(getEnv() != null) { + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); + } observation.setObservationVariableName(variableName); + observation.setValue(value); + + // The BrApi server needs this. Breedbase does not. + BrAPISeason season = new BrAPISeason(); + season.setSeasonDbId(seasonDbId); + observation.setSeason(season); return observation; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a32668134..8f1f16f43 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -38,7 +38,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -92,6 +91,7 @@ public class ExperimentProcessor implements Processor { private BrAPILocationDAO brAPILocationDAO; private BrAPIStudyDAO brAPIStudyDAO; private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private BrAPIObservationDAO brAPIObservationDAO; private BrAPISeasonDAO brAPISeasonDAO; private BrAPIGermplasmDAO brAPIGermplasmDAO; private OntologyService ontologyService; @@ -125,6 +125,7 @@ public ExperimentProcessor(DSLContext dsl, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPIObservationDAO brAPIObservationDAO, BrAPISeasonDAO brAPISeasonDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, OntologyService ontologyService, @@ -134,6 +135,7 @@ public ExperimentProcessor(DSLContext dsl, this.brAPILocationDAO = brAPILocationDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPIObservationDAO = brAPIObservationDAO; this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; this.ontologyService = ontologyService; @@ -200,13 +202,7 @@ public Map process( List tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toList()); // Lookup all traits in system for program, maybe eventually add a variable search in ontology service - List traits = null; - try { - traits = ontologyService.getTraitsByProgramId(program.getId(), true); - } catch (DoesNotExistException e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } + List traits = getTraitList(program); // filter out just traits specified in file List filteredTraits = traits.stream() @@ -277,7 +273,6 @@ public Map process( List> observations = mappedImportRow.getObservations(); // if value was blank won't be entry in map for this observation - PendingImportObject observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))); observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); } @@ -293,7 +288,6 @@ public Map process( throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); } - // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } // End-of-loop @@ -367,14 +361,27 @@ private void getNewBrapiData(List importRows, List> pheno dateTimeValue+="T00:00:00-00:00"; } } - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue); - this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + //column.name() gets phenotype name + String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId); + if(obsPIO.getBrAPIObject() != null && !obsPIO.getBrAPIObject().getValue().isBlank()) { + this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + } } } } +// private String createObservationUnitKey(ExperimentObservation importRow) { +// String key = importRow.getEnv() + importRow.getExpUnitId(); +// return key; +// } + private String createObservationUnitKey(ExperimentObservation importRow) { - String key = importRow.getEnv() + importRow.getExpUnitId(); + String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() ); + return key; + } + private String createObservationUnitKey(String studyName, String obsUnitName) { + String key = studyName + obsUnitName; return key; } @@ -572,18 +579,19 @@ private PendingImportObject createObsUnitPIO(Program progr } - private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue) { + private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId) { PendingImportObject pio = null; if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { pio = observationByHash.get(getImportObservationHash(importRow, variableName)); } else { - BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName); + BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); } return pio; @@ -651,6 +659,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List newLocations = ProcessorData.getNewObjects(this.locationByName); List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + List newObservations = ProcessorData.getNewObjects(this.observationByHash); try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); @@ -687,12 +696,85 @@ public void postBrapiData(Map mappedBrAPIImport, Program } updateObsUnitDependencyValues(program.getKey()); - brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + //brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for( BrAPIObservationUnit createdObservationUnit : createdObservationUnits){ + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getStudyName(),program.getKey() ); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getObservationUnitName(),program.getKey() ); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + PendingImportObject pi = this.observationUnitByNameNoScope.get(createdObsUnit_key); + BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject(); + brAPIObservationUnit.setObservationUnitDbId ( createdObservationUnit.getObservationUnitDbId() ); + } + + updateObservationDependencyValues(program); + brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload); + } catch (ApiException e) { log.error(e.getResponseBody()); throw new InternalServerException(e.toString(), e); } + } + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + // update germplasm DbIds + this.existingGermplasmByGID.values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateObservationGermplasmDbId); + + // update study DbIds and Observation Unit DbIds + this.observationUnitByNameNoScope.values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + for(PendingImportObject observation: this.observationByHash.values()){ + traitsLoop: + for(Trait trait: traits){ + if( observation.getBrAPIObject()!=null && observation.getBrAPIObject().getObservationVariableName()!=null + && observation.getBrAPIObject().getObservationVariableName().equals(trait.getObservationVariableName()) + ){ + observation.getBrAPIObject().setObservationVariableDbId(trait.getObservationVariableDbId()); + break traitsLoop; + } + } + } + } + + private List getTraitList(Program program) { + List traits = null; + try { + traits = ontologyService.getTraitsByProgramId(program.getId(), true); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + return traits; + } + + private void updateObservationGermplasmDbId(BrAPIGermplasm germplasm) { + this.observationByHash.values().stream() + .filter(obs -> obs.getBrAPIObject().getGermplasmName() != null && + obs.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) + .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); + } + + private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { + this.observationByHash.values().stream() + .filter(obs -> obs.getBrAPIObject().getAdditionalInfo() !=null && obs.getBrAPIObject().getAdditionalInfo().get( BrAPIAdditionalInfoFields.STUDY_NAME)!=null && + obs.getBrAPIObject().getAdditionalInfo().get( BrAPIAdditionalInfoFields.STUDY_NAME).getAsString().equals( obsUnit.getStudyName() ) ) + .forEach(obs -> { + obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + }); } private void updateObsUnitDependencyValues(String programKey) { @@ -797,8 +879,6 @@ private Map> initialize_existingGerm private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); - - if( this.trialByNameNoScope.size()!=1){ return studyByNameNoScope; } @@ -990,6 +1070,20 @@ private boolean validCategory(List categories, */ private String yearsToSeasonDbId(List years, UUID programId) { String year = years.get(0); + return this.yearToSeasonDbId(year, programId); + } + + /** + * Converts year String to SeasonDbId + *
+ * NOTE: This assumes that the only Season records of interest are ones + * with a blank name or a name that is the same as the year. + * + * @param year The year as a string + * @param programId the program ID. + * @return the DbId of the season-record associated with the year + */ + private String yearToSeasonDbId(String year, UUID programId) { String dbID = null; if (this.yearToSeasonDbIdCache.containsKey(year) ){ // get it from cache if possible dbID = this.yearToSeasonDbIdCache.get(year); @@ -1001,6 +1095,7 @@ private String yearsToSeasonDbId(List years, UUID programId) { return dbID; } + private String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; if (this.seasonDbIdToYearCache.containsKey(seasonDbId) ){ // get it from cache if possible diff --git a/src/main/java/org/breedinginsight/model/Trait.java b/src/main/java/org/breedinginsight/model/Trait.java index 04611a5df..243f1df40 100644 --- a/src/main/java/org/breedinginsight/model/Trait.java +++ b/src/main/java/org/breedinginsight/model/Trait.java @@ -63,6 +63,7 @@ public class Trait extends TraitEntity { private User updatedByUser; // Properties from brapi + private String observationVariableDbId; private String traitClass; private String traitDescription; private String attribute; @@ -120,7 +121,7 @@ public void setBrAPIProperties(BrAPIObservationVariable brApiVariable) { this.setMainAbbreviation(brApiVariable.getTrait().getMainAbbreviation()); this.setSynonyms(brApiVariable.getTrait().getSynonyms()); } - + this.setObservationVariableDbId(brApiVariable.getObservationVariableDbId()); this.setDefaultValue(brApiVariable.getDefaultValue()); } From ddc548c81a2e2f19820d628f869f50628fd7fcc8 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 22 Dec 2022 10:12:26 -0500 Subject: [PATCH 2/9] [BI-1665] Remove valuless observations just before saving --- .../services/processors/ExperimentProcessor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 8f1f16f43..ebdb05fbb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -364,9 +364,7 @@ private void getNewBrapiData(List importRows, List> pheno //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId); - if(obsPIO.getBrAPIObject() != null && !obsPIO.getBrAPIObject().getValue().isBlank()) { - this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); - } + this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); } } } @@ -659,8 +657,10 @@ public void postBrapiData(Map mappedBrAPIImport, Program List newLocations = ProcessorData.getNewObjects(this.locationByName); List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - List newObservations = ProcessorData.getNewObjects(this.observationByHash); - + // Do not save observations with no 'value' + List newObservations = ProcessorData.getNewObjects(this.observationByHash).stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial From 55691e8d8fc58314ed346df826f1901c86a9df46 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 22 Dec 2022 12:49:47 -0500 Subject: [PATCH 3/9] [BI-1665] Clean up code --- .../services/processors/ExperimentProcessor.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index ebdb05fbb..d479690f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -369,15 +369,11 @@ private void getNewBrapiData(List importRows, List> pheno } } -// private String createObservationUnitKey(ExperimentObservation importRow) { -// String key = importRow.getEnv() + importRow.getExpUnitId(); -// return key; -// } - private String createObservationUnitKey(ExperimentObservation importRow) { - String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() ); - return key; + String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() ); + return key; } + private String createObservationUnitKey(String studyName, String obsUnitName) { String key = studyName + obsUnitName; return key; @@ -657,7 +653,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List newLocations = ProcessorData.getNewObjects(this.locationByName); List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - // Do not save observations with no 'value' + // filter out observations with no 'value' so they will not be saved List newObservations = ProcessorData.getNewObjects(this.observationByHash).stream() .filter(obs -> !obs.getValue().isBlank()) .collect(Collectors.toList()); @@ -696,15 +692,16 @@ public void postBrapiData(Map mappedBrAPIImport, Program } updateObsUnitDependencyValues(program.getKey()); - //brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); // set the DbId to the for each newly created Observation Unit for( BrAPIObservationUnit createdObservationUnit : createdObservationUnits){ + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getStudyName(),program.getKey() ); String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getObservationUnitName(),program.getKey() ); String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); PendingImportObject pi = this.observationUnitByNameNoScope.get(createdObsUnit_key); + // update the retrieved BrAPI object BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject(); brAPIObservationUnit.setObservationUnitDbId ( createdObservationUnit.getObservationUnitDbId() ); } From a977fc04c5b33bb2575dc5675182253108bf8aaf Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 6 Jan 2023 16:39:52 -0500 Subject: [PATCH 4/9] [BI-1665] Fixing setting observationUnitDbId for observations --- .../ExperimentObservation.java | 5 +++- .../processors/ExperimentProcessor.java | 24 ++++++++++++++----- .../utilities/BrAPIDAOUtil.java | 12 +++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index d3e0540dc..2129a457d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -287,7 +287,8 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( public BrAPIObservation constructBrAPIObservation( String value, String variableName, - String seasonDbId + String seasonDbId, + BrAPIObservationUnit obsUnit ) { BrAPIObservation observation = new BrAPIObservation(); observation.setGermplasmDbId(getGid()); @@ -296,6 +297,8 @@ public BrAPIObservation constructBrAPIObservation( observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); } observation.setObservationVariableName(variableName); + observation.setObservationDbId(obsUnit.getObservationUnitDbId()); + observation.setObservationUnitName(obsUnit.getObservationUnitName()); observation.setValue(value); // The BrApi server needs this. Breedbase does not. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d479690f9..b0c5975be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -363,7 +363,7 @@ private void getNewBrapiData(List importRows, List> pheno } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId); + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO); this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); } } @@ -573,13 +573,13 @@ private PendingImportObject createObsUnitPIO(Program progr } - private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId) { + private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId, PendingImportObject obsUnitPIO) { PendingImportObject pio = null; if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { pio = observationByHash.get(getImportObservationHash(importRow, variableName)); } else { - BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId); + BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject()); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { @@ -766,11 +766,23 @@ private void updateObservationGermplasmDbId(BrAPIGermplasm germplasm) { private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { this.observationByHash.values().stream() - .filter(obs -> obs.getBrAPIObject().getAdditionalInfo() !=null && obs.getBrAPIObject().getAdditionalInfo().get( BrAPIAdditionalInfoFields.STUDY_NAME)!=null && - obs.getBrAPIObject().getAdditionalInfo().get( BrAPIAdditionalInfoFields.STUDY_NAME).getAsString().equals( obsUnit.getStudyName() ) ) + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(obsUnit.getStudyName()) + && obs.getBrAPIObject() + .getObservationUnitName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey))) .forEach(obs -> { + if(obs.getBrAPIObject().getObservationUnitDbId() == null) { + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); }); } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 6a302638d..60a6789de 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -190,11 +190,17 @@ public List post(List brapiObjects, progressUpdateMethod.accept(upload); } ApiResponse response = postMethod.apply(postChunk); - if (response.getBody() == null) throw new ApiException("Response is missing body"); + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } BrAPIResponse body = (BrAPIResponse) response.getBody(); - if (body.getResult() == null) throw new ApiException("Response body is missing result"); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } BrAPIResponseResult result = (BrAPIResponseResult) body.getResult(); - if (result.getData() == null) throw new ApiException("Response result is missing data"); + if (result.getData() == null) { + throw new ApiException("Response result is missing data", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } List data = result.getData(); // TODO: Maybe move this outside of the loop if (data.size() != postChunk.size()) throw new ApiException("Number of brapi objects returned does not equal number sent"); From 5f6dcb8167e7867f8cb7c9b42982038789184539 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 11 Jan 2023 09:51:15 -0500 Subject: [PATCH 5/9] [BI-1665] Fixing setting observationUnitDbId for observations --- .../importer/services/processors/ExperimentProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b0c5975be..bee6fdb4e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -775,11 +775,11 @@ private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String program .get(BrAPIAdditionalInfoFields.STUDY_NAME) .getAsString() .equals(obsUnit.getStudyName()) - && obs.getBrAPIObject() - .getObservationUnitName() - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey))) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject() + .getObservationUnitName(), programKey) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey))) .forEach(obs -> { - if(obs.getBrAPIObject().getObservationUnitDbId() == null) { + if(StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); } obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); From 8752e4abcb90a7c4b417f3c4bbe6881c1dd8cd88 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 13 Jan 2023 08:28:09 -0500 Subject: [PATCH 6/9] [BI-1665] Made the Updating of ObservationVariable DbIds more efficient --- .../services/processors/ExperimentProcessor.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index bee6fdb4e..d190ac5f6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -733,15 +733,13 @@ private void updateObservationDependencyValues(Program program) { // Update ObservationVariable DbIds List traits = getTraitList(program); + Map traitMap = traits.stream().collect(Collectors.toMap(trait -> trait.getObservationVariableName(), trait -> trait)); + for(PendingImportObject observation: this.observationByHash.values()){ - traitsLoop: - for(Trait trait: traits){ - if( observation.getBrAPIObject()!=null && observation.getBrAPIObject().getObservationVariableName()!=null - && observation.getBrAPIObject().getObservationVariableName().equals(trait.getObservationVariableName()) - ){ - observation.getBrAPIObject().setObservationVariableDbId(trait.getObservationVariableDbId()); - break traitsLoop; - } + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if( observationVariableName!=null && traitMap.containsKey(observationVariableName)){ + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId( observationVariableDbId ); } } } From 3932f163ee7f60b30367d936e5b3cd063735df14 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 20 Jan 2023 11:58:53 -0500 Subject: [PATCH 7/9] [BI-1665] Move logic to update the observation's gemplasmDbId to the updateObservationDbIds() method --- .../ExperimentObservation.java | 1 - .../processors/ExperimentProcessor.java | 21 +++++-------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 2129a457d..1c50402a2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -291,7 +291,6 @@ public BrAPIObservation constructBrAPIObservation( BrAPIObservationUnit obsUnit ) { BrAPIObservation observation = new BrAPIObservation(); - observation.setGermplasmDbId(getGid()); observation.setGermplasmName(getGermplasmName()); if(getEnv() != null) { observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d190ac5f6..f080ee6f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -708,7 +708,6 @@ public void postBrapiData(Map mappedBrAPIImport, Program updateObservationDependencyValues(program); brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload); - } catch (ApiException e) { log.error(e.getResponseBody()); throw new InternalServerException(e.toString(), e); @@ -717,14 +716,8 @@ public void postBrapiData(Map mappedBrAPIImport, Program private void updateObservationDependencyValues(Program program) { String programKey = program.getKey(); - // update germplasm DbIds - this.existingGermplasmByGID.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateObservationGermplasmDbId); - // update study DbIds and Observation Unit DbIds + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds this.observationUnitByNameNoScope.values().stream() .filter(Objects::nonNull) .distinct() @@ -755,16 +748,11 @@ private List getTraitList(Program program) { return traits; } - private void updateObservationGermplasmDbId(BrAPIGermplasm germplasm) { - this.observationByHash.values().stream() - .filter(obs -> obs.getBrAPIObject().getGermplasmName() != null && - obs.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) - .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); - } - private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { + // Match on Env and Exp Unit ID this.observationByHash.values().stream() - .filter(obs -> obs.getBrAPIObject() + .filter(obs -> + obs.getBrAPIObject() .getAdditionalInfo() != null && obs.getBrAPIObject() .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null @@ -781,6 +769,7 @@ private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String program obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); } obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); }); } From aac6063c2687678ed11f233f32c22980ed78a296 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 20 Jan 2023 12:22:42 -0500 Subject: [PATCH 8/9] [BI-1665] format updateObservationDbIds() --- .../processors/ExperimentProcessor.java | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f080ee6f1..a1d72720c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -748,22 +748,27 @@ private List getTraitList(Program program) { return traits; } + // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { - // Match on Env and Exp Unit ID + // FILTER LOGIC: Match on Env and Exp Unit ID this.observationByHash.values().stream() - .filter(obs -> - obs.getBrAPIObject() - .getAdditionalInfo() != null && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(obsUnit.getStudyName()) - && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject() - .getObservationUnitName(), programKey) - .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey))) + .filter(obs -> + obs.getBrAPIObject().getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(obsUnit.getStudyName()) + && Utilities.removeProgramKeyAndUnknownAdditionalData( + obs.getBrAPIObject().getObservationUnitName(), programKey) + .equals( + Utilities.removeProgramKeyAndUnknownAdditionalData( + obsUnit.getObservationUnitName(), programKey) + ) + ) .forEach(obs -> { if(StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); From 82f4b52fb5e468071bfc967e3f499b39d602c0d6 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 20 Jan 2023 14:01:06 -0500 Subject: [PATCH 9/9] [BI-1665] remove the yearsToSeasonDbId() method --- .../processors/ExperimentProcessor.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a1d72720c..14052f86a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -603,7 +603,8 @@ private PendingImportObject createStudyPIO(Program program, boolean BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); if( commit) { - String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + String seasonID = this.yearToSeasonDbId(year, program.getId()); newStudy.setSeasons(Arrays.asList(seasonID)); } @@ -1058,22 +1059,6 @@ private boolean validCategory(List categories, return categoryValues.contains(value.toLowerCase()); } - /** - * Converts year String to SeasonDbId - *
- * NOTE: This assumes that the only Season records of interest are ones - * with a blank name or a name that is the same as the year. - * - * @param years this only looks at the first year of the list. - * @param programId the program ID. - * @return the DbId of the season-record associated with the first year - * of the 'years' list (see NOTE above) - */ - private String yearsToSeasonDbId(List years, UUID programId) { - String year = years.get(0); - return this.yearToSeasonDbId(year, programId); - } - /** * Converts year String to SeasonDbId *