diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 3968eb7e0..1a633b9a1 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -37,4 +37,5 @@ public final class BrAPIAdditionalInfoFields { public static final String EXPERIMENT_TYPE = "experimentType"; public static final String EXPERIMENT_NUMBER = "experimentNumber"; public static final String ENVIRONMENT_NUMBER = "environmentNumber"; + public static final String STUDY_NAME = "studyName"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 4042b5cc2..1c50402a2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -284,12 +284,26 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } - // TODO: Fill out with rest of data for saving to BRAPI - public BrAPIObservation constructBrAPIObservation(String value, String variableName) { + public BrAPIObservation constructBrAPIObservation( + String value, + String variableName, + String seasonDbId, + BrAPIObservationUnit obsUnit + ) { BrAPIObservation observation = new BrAPIObservation(); - - observation.setValue(value); + observation.setGermplasmName(getGermplasmName()); + if(getEnv() != null) { + observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); + } observation.setObservationVariableName(variableName); + observation.setObservationDbId(obsUnit.getObservationUnitDbId()); + observation.setObservationUnitName(obsUnit.getObservationUnitName()); + observation.setValue(value); + + // The BrApi server needs this. Breedbase does not. + BrAPISeason season = new BrAPISeason(); + season.setSeasonDbId(seasonDbId); + observation.setSeason(season); return observation; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a32668134..14052f86a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -38,7 +38,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -92,6 +91,7 @@ public class ExperimentProcessor implements Processor { private BrAPILocationDAO brAPILocationDAO; private BrAPIStudyDAO brAPIStudyDAO; private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private BrAPIObservationDAO brAPIObservationDAO; private BrAPISeasonDAO brAPISeasonDAO; private BrAPIGermplasmDAO brAPIGermplasmDAO; private OntologyService ontologyService; @@ -125,6 +125,7 @@ public ExperimentProcessor(DSLContext dsl, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPIObservationDAO brAPIObservationDAO, BrAPISeasonDAO brAPISeasonDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, OntologyService ontologyService, @@ -134,6 +135,7 @@ public ExperimentProcessor(DSLContext dsl, this.brAPILocationDAO = brAPILocationDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPIObservationDAO = brAPIObservationDAO; this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; this.ontologyService = ontologyService; @@ -200,13 +202,7 @@ public Map process( List tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toList()); // Lookup all traits in system for program, maybe eventually add a variable search in ontology service - List traits = null; - try { - traits = ontologyService.getTraitsByProgramId(program.getId(), true); - } catch (DoesNotExistException e) { - log.error(e.getMessage(), e); - throw new InternalServerException(e.toString(), e); - } + List traits = getTraitList(program); // filter out just traits specified in file List filteredTraits = traits.stream() @@ -277,7 +273,6 @@ public Map process( List> observations = mappedImportRow.getObservations(); // if value was blank won't be entry in map for this observation - PendingImportObject observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))); observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); } @@ -293,7 +288,6 @@ public Map process( throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); } - // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } // End-of-loop @@ -367,14 +361,21 @@ private void getNewBrapiData(List importRows, List> pheno dateTimeValue+="T00:00:00-00:00"; } } - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue); + //column.name() gets phenotype name + String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO); this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); } } } private String createObservationUnitKey(ExperimentObservation importRow) { - String key = importRow.getEnv() + importRow.getExpUnitId(); + String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() ); + return key; + } + + private String createObservationUnitKey(String studyName, String obsUnitName) { + String key = studyName + obsUnitName; return key; } @@ -572,18 +573,19 @@ private PendingImportObject createObsUnitPIO(Program progr } - private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue) { + private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId, PendingImportObject obsUnitPIO) { PendingImportObject pio = null; if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { pio = observationByHash.get(getImportObservationHash(importRow, variableName)); } else { - BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName); + BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject()); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); } return pio; @@ -601,7 +603,8 @@ private PendingImportObject createStudyPIO(Program program, boolean BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); if( commit) { - String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + String seasonID = this.yearToSeasonDbId(year, program.getId()); newStudy.setSeasons(Arrays.asList(seasonID)); } @@ -651,7 +654,10 @@ public void postBrapiData(Map mappedBrAPIImport, Program List newLocations = ProcessorData.getNewObjects(this.locationByName); List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash).stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial @@ -687,12 +693,90 @@ public void postBrapiData(Map mappedBrAPIImport, Program } updateObsUnitDependencyValues(program.getKey()); - brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for( BrAPIObservationUnit createdObservationUnit : createdObservationUnits){ + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getStudyName(),program.getKey() ); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getObservationUnitName(),program.getKey() ); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + PendingImportObject pi = this.observationUnitByNameNoScope.get(createdObsUnit_key); + // update the retrieved BrAPI object + BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject(); + brAPIObservationUnit.setObservationUnitDbId ( createdObservationUnit.getObservationUnitDbId() ); + } + + updateObservationDependencyValues(program); + brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload); } catch (ApiException e) { log.error(e.getResponseBody()); throw new InternalServerException(e.toString(), e); } + } + + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + Map traitMap = traits.stream().collect(Collectors.toMap(trait -> trait.getObservationVariableName(), trait -> trait)); + + for(PendingImportObject observation: this.observationByHash.values()){ + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if( observationVariableName!=null && traitMap.containsKey(observationVariableName)){ + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId( observationVariableDbId ); + } + } + } + + private List getTraitList(Program program) { + List traits = null; + try { + traits = ontologyService.getTraitsByProgramId(program.getId(), true); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + return traits; + } + // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId + private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { + // FILTER LOGIC: Match on Env and Exp Unit ID + this.observationByHash.values().stream() + .filter(obs -> + obs.getBrAPIObject().getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(obsUnit.getStudyName()) + && Utilities.removeProgramKeyAndUnknownAdditionalData( + obs.getBrAPIObject().getObservationUnitName(), programKey) + .equals( + Utilities.removeProgramKeyAndUnknownAdditionalData( + obsUnit.getObservationUnitName(), programKey) + ) + ) + .forEach(obs -> { + if(StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); + }); } private void updateObsUnitDependencyValues(String programKey) { @@ -797,8 +881,6 @@ private Map> initialize_existingGerm private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); - - if( this.trialByNameNoScope.size()!=1){ return studyByNameNoScope; } @@ -983,13 +1065,11 @@ private boolean validCategory(List categories, * NOTE: This assumes that the only Season records of interest are ones * with a blank name or a name that is the same as the year. * - * @param years this only looks at the first year of the list. + * @param year The year as a string * @param programId the program ID. - * @return the DbId of the season-record associated with the first year - * of the 'years' list (see NOTE above) + * @return the DbId of the season-record associated with the year */ - private String yearsToSeasonDbId(List years, UUID programId) { - String year = years.get(0); + private String yearToSeasonDbId(String year, UUID programId) { String dbID = null; if (this.yearToSeasonDbIdCache.containsKey(year) ){ // get it from cache if possible dbID = this.yearToSeasonDbIdCache.get(year); @@ -1001,6 +1081,7 @@ private String yearsToSeasonDbId(List years, UUID programId) { return dbID; } + private String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; if (this.seasonDbIdToYearCache.containsKey(seasonDbId) ){ // get it from cache if possible diff --git a/src/main/java/org/breedinginsight/model/Trait.java b/src/main/java/org/breedinginsight/model/Trait.java index 04611a5df..243f1df40 100644 --- a/src/main/java/org/breedinginsight/model/Trait.java +++ b/src/main/java/org/breedinginsight/model/Trait.java @@ -63,6 +63,7 @@ public class Trait extends TraitEntity { private User updatedByUser; // Properties from brapi + private String observationVariableDbId; private String traitClass; private String traitDescription; private String attribute; @@ -120,7 +121,7 @@ public void setBrAPIProperties(BrAPIObservationVariable brApiVariable) { this.setMainAbbreviation(brApiVariable.getTrait().getMainAbbreviation()); this.setSynonyms(brApiVariable.getTrait().getSynonyms()); } - + this.setObservationVariableDbId(brApiVariable.getObservationVariableDbId()); this.setDefaultValue(brApiVariable.getDefaultValue()); } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 6a302638d..60a6789de 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -190,11 +190,17 @@ public List post(List brapiObjects, progressUpdateMethod.accept(upload); } ApiResponse response = postMethod.apply(postChunk); - if (response.getBody() == null) throw new ApiException("Response is missing body"); + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } BrAPIResponse body = (BrAPIResponse) response.getBody(); - if (body.getResult() == null) throw new ApiException("Response body is missing result"); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } BrAPIResponseResult result = (BrAPIResponseResult) body.getResult(); - if (result.getData() == null) throw new ApiException("Response result is missing data"); + if (result.getData() == null) { + throw new ApiException("Response result is missing data", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } List data = result.getData(); // TODO: Maybe move this outside of the loop if (data.size() != postChunk.size()) throw new ApiException("Number of brapi objects returned does not equal number sent");