Skip to content

BI-1665 Store observations when importing new environments #234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davedrp merged 9 commits into from
Feb 2, 2023
Merged

BI-1665 Store observations when importing new environments #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1595a20
[BI-1665] Save each observation to the BrAPI backend
davedrp Dec 13, 2022
ddc548c
[BI-1665] Remove valuless observations just before saving
davedrp Dec 22, 2022
55691e8
[BI-1665] Clean up code
davedrp Dec 22, 2022
a977fc0
[BI-1665] Fixing setting observationUnitDbId for observations
timparsons Jan 6, 2023
5f6dcb8
[BI-1665] Fixing setting observationUnitDbId for observations
timparsons Jan 11, 2023
8752e4a
[BI-1665] Made the Updating of ObservationVariable DbIds more efficient
davedrp Jan 13, 2023
3932f16
[BI-1665] Move logic to update the observation's gemplasmDbId to the …
davedrp Jan 20, 2023
aac6063
[BI-1665] format updateObservationDbIds()
davedrp Jan 20, 2023
82f4b52
[BI-1665] remove the yearsToSeasonDbId() method
davedrp Jan 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,5 @@ public final class BrAPIAdditionalInfoFields {
public static final String EXPERIMENT_TYPE = "experimentType";
public static final String EXPERIMENT_NUMBER = "experimentNumber";
public static final String ENVIRONMENT_NUMBER = "environmentNumber";
public static final String STUDY_NAME = "studyName";
}
22 changes: 18 additions & 4 deletions ...inginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,12 +284,26 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(
return observationUnit;
}

// TODO: Fill out with rest of data for saving to BRAPI
public BrAPIObservation constructBrAPIObservation(String value, String variableName) {
public BrAPIObservation constructBrAPIObservation(
String value,
String variableName,
String seasonDbId,
BrAPIObservationUnit obsUnit
) {
BrAPIObservation observation = new BrAPIObservation();

observation.setValue(value);
observation.setGermplasmName(getGermplasmName());
if(getEnv() != null) {
observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv());
}
observation.setObservationVariableName(variableName);
observation.setObservationDbId(obsUnit.getObservationUnitDbId());
observation.setObservationUnitName(obsUnit.getObservationUnitName());
observation.setValue(value);

// The BrApi server needs this. Breedbase does not.
BrAPISeason season = new BrAPISeason();
season.setSeasonDbId(seasonDbId);
observation.setSeason(season);

return observation;
}
Expand Down
129 changes: 105 additions & 24 deletions ...ain/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO;
import org.breedinginsight.brapps.importer.daos.*;
import org.breedinginsight.brapps.importer.model.ImportUpload;
import org.breedinginsight.brapps.importer.model.base.Observation;
import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation;
import org.breedinginsight.brapps.importer.model.imports.BrAPIImport;
import org.breedinginsight.brapps.importer.model.imports.PendingImport;
Expand Down Expand Up @@ -92,6 +91,7 @@ public class ExperimentProcessor implements Processor {
private BrAPILocationDAO brAPILocationDAO;
private BrAPIStudyDAO brAPIStudyDAO;
private BrAPIObservationUnitDAO brAPIObservationUnitDAO;
private BrAPIObservationDAO brAPIObservationDAO;
private BrAPISeasonDAO brAPISeasonDAO;
private BrAPIGermplasmDAO brAPIGermplasmDAO;
private OntologyService ontologyService;
Expand Down Expand Up @@ -125,6 +125,7 @@ public ExperimentProcessor(DSLContext dsl,
BrAPILocationDAO brAPILocationDAO,
BrAPIStudyDAO brAPIStudyDAO,
BrAPIObservationUnitDAO brAPIObservationUnitDAO,
BrAPIObservationDAO brAPIObservationDAO,
BrAPISeasonDAO brAPISeasonDAO,
BrAPIGermplasmDAO brAPIGermplasmDAO,
OntologyService ontologyService,
Expand All @@ -134,6 +135,7 @@ public ExperimentProcessor(DSLContext dsl,
this.brAPILocationDAO = brAPILocationDAO;
this.brAPIStudyDAO = brAPIStudyDAO;
this.brAPIObservationUnitDAO = brAPIObservationUnitDAO;
this.brAPIObservationDAO = brAPIObservationDAO;
this.brAPISeasonDAO = brAPISeasonDAO;
this.brAPIGermplasmDAO = brAPIGermplasmDAO;
this.ontologyService = ontologyService;
Expand Down Expand Up @@ -200,13 +202,7 @@ public Map<String, ImportPreviewStatistics> process(
List<String> tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toList());

// Lookup all traits in system for program, maybe eventually add a variable search in ontology service
List<Trait> traits = null;
try {
traits = ontologyService.getTraitsByProgramId(program.getId(), true);
} catch (DoesNotExistException e) {
log.error(e.getMessage(), e);
throw new InternalServerException(e.toString(), e);
}
List<Trait> traits = getTraitList(program);

// filter out just traits specified in file
List<Trait> filteredTraits = traits.stream()
Expand Down Expand Up @@ -277,7 +273,6 @@ public Map<String, ImportPreviewStatistics> process(
List<PendingImportObject<BrAPIObservation>> observations = mappedImportRow.getObservations();

// if value was blank won't be entry in map for this observation
PendingImportObject<BrAPIObservation> observation = this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)));
observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column))));
}

Expand All @@ -293,7 +288,6 @@ public Map<String, ImportPreviewStatistics> process(
throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR);
}

// Construct Observations -- Done in another card
mappedBrAPIImport.put(i, mappedImportRow);
}
// End-of-loop
Expand Down Expand Up @@ -367,14 +361,21 @@ private void getNewBrapiData(List<BrAPIImport> importRows, List<Column<?>> pheno
dateTimeValue+="T00:00:00-00:00";
}
}
PendingImportObject<BrAPIObservation> obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue);
//column.name() gets phenotype name
String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId());
PendingImportObject<BrAPIObservation> obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO);
this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO);
}
}
}

private String createObservationUnitKey(ExperimentObservation importRow) {
String key = importRow.getEnv() + importRow.getExpUnitId();
String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() );
return key;
}

private String createObservationUnitKey(String studyName, String obsUnitName) {
String key = studyName + obsUnitName;
return key;
}

Expand Down Expand Up @@ -572,18 +573,19 @@ private PendingImportObject<BrAPIObservationUnit> createObsUnitPIO(Program progr
}


private PendingImportObject<BrAPIObservation> createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue) {
private PendingImportObject<BrAPIObservation> createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId, PendingImportObject<BrAPIObservationUnit> obsUnitPIO) {
PendingImportObject<BrAPIObservation> pio = null;
if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) {
pio = observationByHash.get(getImportObservationHash(importRow, variableName));
}
else {
BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName);
BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject());
//NOTE: Can't parse invalid timestamp value, so have to skip if invalid.
// Validation error should be thrown for offending value, but that doesn't happen until later downstream
if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) {
newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue));
}

pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation);
}
return pio;
Expand All @@ -601,7 +603,8 @@ private PendingImportObject<BrAPIStudy> createStudyPIO(Program program, boolean
BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal);

if( commit) {
String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId());
String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season
String seasonID = this.yearToSeasonDbId(year, program.getId());
newStudy.setSeasons(Arrays.asList(seasonID));
}

Expand Down Expand Up @@ -651,7 +654,10 @@ public void postBrapiData(Map<Integer, PendingImport> mappedBrAPIImport, Program
List<BrAPILocation> newLocations = ProcessorData.getNewObjects(this.locationByName);
List<BrAPIStudy> newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope);
List<BrAPIObservationUnit> newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope);

// filter out observations with no 'value' so they will not be saved
List<BrAPIObservation> newObservations = ProcessorData.getNewObjects(this.observationByHash).stream()
.filter(obs -> !obs.getValue().isBlank())
.collect(Collectors.toList());
try {
List<BrAPITrial> createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload));
// set the DbId to the for each newly created trial
Expand Down Expand Up @@ -687,12 +693,90 @@ public void postBrapiData(Map<Integer, PendingImport> mappedBrAPIImport, Program
}

updateObsUnitDependencyValues(program.getKey());
brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload);
List<BrAPIObservationUnit> createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload);

// set the DbId to the for each newly created Observation Unit
for( BrAPIObservationUnit createdObservationUnit : createdObservationUnits){
// retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope
String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getStudyName(),program.getKey() );
String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getObservationUnitName(),program.getKey() );
String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName);
PendingImportObject<BrAPIObservationUnit> pi = this.observationUnitByNameNoScope.get(createdObsUnit_key);
// update the retrieved BrAPI object
BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject();
brAPIObservationUnit.setObservationUnitDbId ( createdObservationUnit.getObservationUnitDbId() );
}

updateObservationDependencyValues(program);
brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload);
} catch (ApiException e) {
log.error(e.getResponseBody());
throw new InternalServerException(e.toString(), e);
}
}

private void updateObservationDependencyValues(Program program) {
String programKey = program.getKey();

// update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds
this.observationUnitByNameNoScope.values().stream()
.filter(Objects::nonNull)
.distinct()
.map(PendingImportObject::getBrAPIObject)
.forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey));

// Update ObservationVariable DbIds
List<Trait> traits = getTraitList(program);
Map<String, Trait> traitMap = traits.stream().collect(Collectors.toMap(trait -> trait.getObservationVariableName(), trait -> trait));

for(PendingImportObject<BrAPIObservation> observation: this.observationByHash.values()){
String observationVariableName = observation.getBrAPIObject().getObservationVariableName();
if( observationVariableName!=null && traitMap.containsKey(observationVariableName)){
String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId();
observation.getBrAPIObject().setObservationVariableDbId( observationVariableDbId );
}
}
}

private List<Trait> getTraitList(Program program) {
List<Trait> traits = null;
try {
traits = ontologyService.getTraitsByProgramId(program.getId(), true);
} catch (DoesNotExistException e) {
log.error(e.getMessage(), e);
throw new InternalServerException(e.toString(), e);
}
return traits;
}

// Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId
private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) {
// FILTER LOGIC: Match on Env and Exp Unit ID
this.observationByHash.values().stream()
.filter(obs ->
obs.getBrAPIObject().getAdditionalInfo() != null
&& obs.getBrAPIObject()
.getAdditionalInfo()
.get(BrAPIAdditionalInfoFields.STUDY_NAME) != null
&& obs.getBrAPIObject()
.getAdditionalInfo()
.get(BrAPIAdditionalInfoFields.STUDY_NAME)
.getAsString()
.equals(obsUnit.getStudyName())
&& Utilities.removeProgramKeyAndUnknownAdditionalData(
obs.getBrAPIObject().getObservationUnitName(), programKey)
.equals(
Utilities.removeProgramKeyAndUnknownAdditionalData(
obsUnit.getObservationUnitName(), programKey)
)
)
.forEach(obs -> {
if(StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) {
obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId());
}
obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId());
obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId());
});
}

private void updateObsUnitDependencyValues(String programKey) {
Expand Down Expand Up @@ -797,8 +881,6 @@ private Map<String, PendingImportObject<BrAPIGermplasm>> initialize_existingGerm

private Map<String, PendingImportObject<BrAPIStudy>> initialize_studyByNameNoScope(Program program, List<ExperimentObservation> experimentImportRows) {
Map<String, PendingImportObject<BrAPIStudy>> studyByNameNoScope = new HashMap<>();


if( this.trialByNameNoScope.size()!=1){
return studyByNameNoScope;
}
Expand Down Expand Up @@ -983,13 +1065,11 @@ private boolean validCategory(List<BrAPIScaleValidValuesCategories> categories,
* NOTE: This assumes that the only Season records of interest are ones
* with a blank name or a name that is the same as the year.
*
* @param years this only looks at the first year of the list.
* @param year The year as a string
* @param programId the program ID.
* @return the DbId of the season-record associated with the first year
* of the 'years' list (see NOTE above)
* @return the DbId of the season-record associated with the year
*/
private String yearsToSeasonDbId(List<String> years, UUID programId) {
String year = years.get(0);
private String yearToSeasonDbId(String year, UUID programId) {
String dbID = null;
if (this.yearToSeasonDbIdCache.containsKey(year) ){ // get it from cache if possible
dbID = this.yearToSeasonDbIdCache.get(year);
Expand All @@ -1001,6 +1081,7 @@ private String yearsToSeasonDbId(List<String> years, UUID programId) {
return dbID;
}


private String seasonDbIdToYear(String seasonDbId, UUID programId) {
String year = null;
if (this.seasonDbIdToYearCache.containsKey(seasonDbId) ){ // get it from cache if possible
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/breedinginsight/model/Trait.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public class Trait extends TraitEntity {
private User updatedByUser;

// Properties from brapi
private String observationVariableDbId;
private String traitClass;
private String traitDescription;
private String attribute;
Expand Down Expand Up @@ -120,7 +121,7 @@ public void setBrAPIProperties(BrAPIObservationVariable brApiVariable) {
this.setMainAbbreviation(brApiVariable.getTrait().getMainAbbreviation());
this.setSynonyms(brApiVariable.getTrait().getSynonyms());
}

this.setObservationVariableDbId(brApiVariable.getObservationVariableDbId());
this.setDefaultValue(brApiVariable.getDefaultValue());
}

Expand Down
12 changes: 9 additions & 3 deletions src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,11 +190,17 @@ public <T> List<T> post(List<T> brapiObjects,
progressUpdateMethod.accept(upload);
}
ApiResponse response = postMethod.apply(postChunk);
if (response.getBody() == null) throw new ApiException("Response is missing body");
if (response.getBody() == null) {
throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null);
}
BrAPIResponse body = (BrAPIResponse) response.getBody();
if (body.getResult() == null) throw new ApiException("Response body is missing result");
if (body.getResult() == null) {
throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString());
}
BrAPIResponseResult result = (BrAPIResponseResult) body.getResult();
if (result.getData() == null) throw new ApiException("Response result is missing data");
if (result.getData() == null) {
throw new ApiException("Response result is missing data", response.getStatusCode(), response.getHeaders(), response.getBody().toString());
}
List<T> data = result.getData();
// TODO: Maybe move this outside of the loop
if (data.size() != postChunk.size()) throw new ApiException("Number of brapi objects returned does not equal number sent");
Expand Down