diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index e603a7d21..cb93daf48 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -27,10 +27,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Singleton public class FileMappingUtil { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 4ea10c07a..cbc7299d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; +import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; @@ -118,23 +119,30 @@ public class ExperimentProcessor implements Processor { private final Map seasonDbIdToYearCache = new HashMap<>(); //These BrapiData-objects are initially populated by the getExistingBrapiData() method, - // then updated by the getNewBrapiData() method. - private Map> trialByNameNoScope = null; + // then updated by the initNewBrapiData() method. + private Map> trialByNameNoScope = new HashMap<>(); + private Map> pendingTrialByOUId = new HashMap<>(); private Map> locationByName = null; - private Map> studyByNameNoScope = null; + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> studyByNameNoScope = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); private Map> obsVarDatasetByName = null; - // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be - // initialized by getExistingBrapiData() ) + private Map> pendingObsDatasetByOUId = new HashMap<>(); private Map> observationUnitByNameNoScope = null; + private Map> pendingObsUnitByOUId = new HashMap<>(); private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); - // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method - private Map> existingGermplasmByGID = null; + // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method + private Map> existingGermplasmByGID = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); // Associates timestamp columns to associated phenotype column name for ease of storage private final Map> timeStampColByPheno = new HashMap<>(); private final Gson gson; + private boolean hasAllReferenceUnitIds = true; + private boolean hasNoReferenceUnitIds = true; + private Set referenceOUIds = new HashSet<>(); @Inject public ExperimentProcessor(DSLContext dsl, @@ -179,12 +187,50 @@ public void getExistingBrapiData(List importRows, Program program) .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); - this.observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - this.trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - this.studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); - this.obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); - this.existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + // check for references to Deltabreed-generated observation units + referenceOUIds = collateReferenceOUIds(importRows); + + if (hasAllReferenceUnitIds) { + try { + + // get all prior units referenced in import + pendingObsUnitByOUId = fetchReferenceObservationUnits(referenceOUIds, program); + observationUnitByNameNoScope = mapPendingObservationUnitByName(pendingObsUnitByOUId, program); + initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); + initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); + locationByName = initializeLocationByName(program, studyByNameNoScope); + obsVarDatasetByName = initializeObsVarDatasetForExistingObservationUnits(trialByNameNoScope, program); + existingGermplasmByGID = initializeGermplasmByGIDForExistingObservationUnits(observationUnitByNameNoScope, program); + for (Map.Entry> unitEntry : pendingObsUnitByOUId.entrySet()) { + String unitId = unitEntry.getKey(); + BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); + mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); + mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); + mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); + mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId); + mapGermplasmByOUId(unitId, unit, existingGermplasmByGID, pendingGermplasmByOUId); + } + + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing experiment with ", e); + throw new InternalServerException(e.toString(), e); + } + } else if (hasNoReferenceUnitIds) { + observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); + studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); + locationByName = initializeUniqueLocationNames(program, experimentImportRows); + obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); + existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + + } else { + + // can't proceed if the import has a mix of ObsUnitId for some but not all rows + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); + } } /** @@ -205,7 +251,7 @@ public Map process( Table data, Program program, User user, - boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException { + boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -416,22 +462,44 @@ public void postBrapiData(Map mappedBrAPIImport, Program private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); - mappedImportRow.setTrial(this.trialByNameNoScope.get(importRow.getExpTitle())); - mappedImportRow.setLocation(this.locationByName.get(importRow.getEnvLocation())); - mappedImportRow.setStudy(this.studyByNameNoScope.get(importRow.getEnv())); - mappedImportRow.setObservationUnit(this.observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); - - // loop over phenotype column observation data for current row List> observations = mappedImportRow.getObservations(); - for (Column column : phenotypeCols) { - // if value was blank won't be entry in map for this observation - observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); - PendingImportObject germplasmPIO = getGidPOI(importRow); - mappedImportRow.setGermplasm(germplasmPIO); + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } mappedBrAPIImport.put(rowNum, mappedImportRow); } @@ -495,7 +563,14 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws ApiException, MissingRequiredInfoException { + private void initNewBrapiData( + List importRows, + List> phenotypeCols, + Program program, + User user, + List referencedTraits, + boolean commit + ) throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -511,6 +586,7 @@ private void initNewBrapiData(List importRows, List> phen } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -557,9 +633,23 @@ private void initNewBrapiData(List importRows, List> phen dateTimeValue += MIDNIGHT; } } - //column.name() gets phenotype name - String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - fetchOrCreateObservationPIO(program, user, importRow, column, rowNum, dateTimeValue, commit, seasonDbId, obsUnitPIO, referencedTraits); + + // get the study year either referenced from the observation unit or listed explicitly on the import row + String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); + String seasonDbId = yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); } } } @@ -572,6 +662,21 @@ private String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + /** + * This method is responsible for generating a hash based on the import observation unit information. + * It takes the observation unit name, variable name, and study name as input parameters. + * The observation unit key is created using the study name and observation unit name. + * The hash is generated based on the observation unit key, variable name, and study name. + * + * @param obsUnitName The name of the observation unit being imported. + * @param variableName The name of the variable associated with the observation unit. + * @param studyName The name of the study associated with the observation unit. + * @return A string representing the hash of the import observation unit information. + */ + private String getImportObservationHash(String obsUnitName, String variableName, String studyName) { + return getObservationHash(createObservationUnitKey(studyName, obsUnitName), variableName, studyName); + } + private String getImportObservationHash(ExperimentObservation importRow, String variableName) { return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); } @@ -594,24 +699,26 @@ private void validateFields(List importRows, ValidationErrors valid for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } - validateTestOrCheck(importRow, validationErrors, rowNum); - //TODO: providing obs unit ID does not supersede import row inout data as expected and needs to be fixed - //Check if existing environment. If so, ObsUnitId must be assigned -// if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) -// && (StringUtils.isBlank(importRow.getObsUnitID()))) { -// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); -// } - - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); - validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } } - private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { + private void validateObservationUnits( + ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow + ) { validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); String key = createObservationUnitKey(importRow); @@ -673,9 +780,20 @@ private void validateObservations(ValidationErrors validationErrors, boolean commit, User user) { phenotypeCols.forEach(phenoCol -> { - String importHash = getImportObservationHash(importRow, phenoCol.name()); + String importHash; String importObsValue = phenoCol.getString(rowNum); + if (hasAllReferenceUnitIds) { + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + } else { + importHash = getImportObservationHash(importRow, phenoCol.name()); + } + // error if import observation data already exists and user has not selected to overwrite if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && this.existingObsByObsHash.containsKey(importHash) && @@ -725,14 +843,17 @@ private void validateObservations(ValidationErrors validationErrors, pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); } - // preview case where observation has already been committed and import ObsVar data is either empty or the - // same as has been committed prior to import - } else if(existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || - isObservationMatched(importHash, importObsValue, phenoCol, rowNum))) { + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if(isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if(!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); } else { validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); @@ -810,8 +931,14 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); } } else { - //TODO: include this step once user-supplied obs unit id correctly supersedes other row data - //validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + Columns.OBS_UNIT_ID, + MISSING_OBS_UNIT_ID_ERROR, + validationErrors, + rowNum + ); } } @@ -922,7 +1049,7 @@ private void validateTestOrCheck(ExperimentObservation importRow, ValidationErro } } - private PendingImportObject getGidPOI(ExperimentObservation importRow) { + private PendingImportObject getGidPIO(ExperimentObservation importRow) { if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { return existingGermplasmByGID.get(importRow.getGid()); } @@ -930,10 +1057,12 @@ private PendingImportObject getGidPOI(ExperimentObservation impo return null; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException { + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { PendingImportObject pio; String key = createObservationUnitKey(importRow); - if (this.observationUnitByNameNoScope.containsKey(key)) { + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; @@ -942,7 +1071,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra .getBrAPIObject() .getGermplasmName(); } - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { @@ -981,7 +1110,7 @@ boolean isTimestampMatched(String observationHash, String timeStamp) { } boolean isValueMatched(String observationHash, String value) { - if (existingObsByObsHash.get(observationHash).getValue() == null) { + if (!existingObsByObsHash.containsKey(observationHash) || existingObsByObsHash.get(observationHash).getValue() == null) { return value == null; } return existingObsByObsHash.get(observationHash).getValue().equals(value); @@ -1005,15 +1134,23 @@ private void fetchOrCreateObservationPIO(Program program, boolean commit, String seasonDbId, PendingImportObject obsUnitPIO, - List referencedTraits) throws ApiException { + PendingImportObject studyPIO, + List referencedTraits) throws ApiException, UnprocessableEntityException { PendingImportObject pio; BrAPIObservation newObservation; String variableName = column.name(); String value = column.getString(rowNum); - String key = getImportObservationHash(importRow, variableName); + String key; + if (hasAllReferenceUnitIds) { + String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); + String studyName = studyPIO.getBrAPIObject().getStudyName(); + key = getObservationHash(studyName + unitName, variableName, studyName); + } else { + key = getImportObservationHash(importRow, variableName); + } if (existingObsByObsHash.containsKey(key)) { - if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, column, rowNum)){ + if (!isObservationMatched(key, value, column, rowNum)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); @@ -1035,9 +1172,10 @@ private void fetchOrCreateObservationPIO(Program program, } else if (!this.observationByHash.containsKey(key)){ // new observation - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); @@ -1047,7 +1185,7 @@ private void fetchOrCreateObservationPIO(Program program, newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } - newObservation.setStudyDbId(this.studyByNameNoScope.get(importRow.getEnv()).getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); @@ -1069,9 +1207,10 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } }); } - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { PendingImportObject pio; - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); String name = String.format("Observation Dataset [%s-%s]", program.getKey(), trialPIO.getBrAPIObject() @@ -1103,15 +1242,26 @@ private PendingImportObject fetchOrCreateStudyPIO( boolean commit, String expSequenceValue, ExperimentObservation importRow, - Supplier envNextVal) { + Supplier envNextVal + ) throws UnprocessableEntityException { PendingImportObject pio; - if (studyByNameNoScope.containsKey(importRow.getEnv())) { + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); - if (! commit){ + if (!commit){ addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); } } else { - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); @@ -1167,33 +1317,53 @@ private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Str private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { PendingImportObject pio; - if (! locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = importRow.constructProgramLocation(); + String envLocationName = hasAllReferenceUnitIds ? + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(importRow.getEnvLocation(), pio); + this.locationByName.put(envLocationName, pio); } } - private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) throws UnprocessableEntityException { + private PendingImportObject fetchOrCreateTrialPIO( + Program program, + User user, + boolean commit, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException { PendingImportObject trialPio; - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = this.studyByNameNoScope.get(importRow.getEnv()); - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); - } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + + // use the prior trial if observation unit IDs are supplied + if (hasAllReferenceUnitIds) { + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + // otherwise create a new trial, but there can be only one allowed } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - this.trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } return trialPio; } @@ -1380,7 +1550,7 @@ private Map> initializeObserva BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID()); + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); @@ -1398,6 +1568,161 @@ private Map> initializeObserva } } + /** + * Maps pending locations by Observation Unit (OU) Id based on given parameters. + * + * This method takes in a unitId, BrAPIObservationUnit unit, Maps of studyByOUId, locationByName, + * and locationByOUId. It then associates the location of the observation unit with the respective OU Id. + * If the locationName is not null for the unit, it is directly added to locationByOUId. + * If the locationName is null, it checks the studyByOUId map for a location related to the unit. + * If a location related to the unit is found, it maps that location with the respective OU Id. + * If no location is found, it throws an IllegalStateException. + * + * @param unitId the Observation Unit Id + * @param unit the BrAPIObservationUnit object + * @param studyByOUId a Map of Study by Observation Unit Id + * @param locationByName a Map of Location by Name + * @param locationByOUId a Map of Location by Observation Unit Id + * @return the updated locationByOUId map after mapping the pending locations + * @throws IllegalStateException if the Observation Unit is missing a location + */ + private Map> mapPendingLocationByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByOUId, + Map> locationByName, + Map> locationByOUId + ) { + if (unit.getLocationName() != null) { + locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); + } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { + locationByOUId.put( + unitId, + locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) + ); + } else { + throw new IllegalStateException("Observation unit missing location: " + unitId); + } + + return locationByOUId; + } + + private Map> mapPendingStudyByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByName, + Map> studyByOUId, + Program program + ) { + if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + studyByOUId.put(unitId, studyByName.get(studyName)); + } else { + throw new IllegalStateException("Observation unit missing study name: " + unitId); + } + + return studyByOUId; + } + private Map> mapPendingTrialByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> trialByName, + Map> studyByName, + Map> trialByOUId, + Program program + ) { + String trialName; + if (unit.getTrialName() != null) { + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); + } else if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( + studyByName.get(studyName).getBrAPIObject().getTrialName(), + program.getKey() + ); + } else { + throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); + } + trialByOUId.put(unitId, trialByName.get(trialName)); + + return trialByOUId; + } + private Map> mapPendingObservationUnitByName( + Map> pendingUnitById, + Program program + ) { + Map> pendingUnitByName = new HashMap<>(); + for (Map.Entry> entry : pendingUnitById.entrySet()) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByName.put(createObservationUnitKey(studyName, observationUnitName), entry.getValue()); + } + return pendingUnitByName; + } + + /** + * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. + * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. + * + * @param referenceOUIds A set of reference Observation Unit IDs to retrieve + * @param program The Program associated with the Observation Units + * @return A Map containing pending Observation Units by their ID + * @throws ApiException if an error occurs during the process + */ + private Map> fetchReferenceObservationUnits( + Set referenceOUIds, + Program program + ) throws ApiException { + Map> pendingUnitById = new HashMap<>(); + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(referenceOUIds), + program + ); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == referenceOUIds.size()) { + // Iterate through reference Observation Units + referenceObsUnits.forEach(unit -> { + // Get external reference for the Observation Unit + BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); + + // Set pending Observation Unit by its ID + pendingUnitById.put( + unitXref.getReferenceId(), + new PendingImportObject( + ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) + ); + }); + } else { + // Handle missing Observation Unit IDs + List missingIds = new ArrayList<>(referenceOUIds); + Set fetchedIds = referenceObsUnits.stream().map(unit -> + Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) + .getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return pendingUnitById; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new ApiException(e); + } + } + private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { Map> trialByName = new HashMap<>(); @@ -1409,9 +1734,9 @@ private Map> initializeTrialByNameNoScop .distinct() .collect(Collectors.toList()); try { - String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - brapiTrialDAO.getTrialsByName(uniqueTrialNames, program) - .forEach(existingTrial -> processAndCacheTrial(existingTrial, program, trialRefSource, trialByName)); + brapiTrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1431,6 +1756,9 @@ private Map> initializeStudyByNameNoScop } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); } List existingStudies; @@ -1442,36 +1770,97 @@ private Map> initializeStudyByNameNoScop } UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); + for (BrAPIStudy existingStudy : existingStudies) { + processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); } return studyByName; } - private void initializeStudiesForExistingObservationUnits(Program program, Map> studyByName) throws ApiException { - Set studyDbIds = this.observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); List studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> processAndCacheStudy(study, program, studyByName)); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } } + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if(studies.size() != studyDbIds.size()) { + if (studies.size() != studyDbIds.size()) { List missingIds = new ArrayList<>(studyDbIds); missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); } return studies; } + /** + * Initializes a map of ProgramLocation objects by their names using the given Program and a map of BrAPIStudy objects by their names. + * + * This method takes a Program object and a map of BrAPIStudy objects by their names, retrieves the location database IDs from the studies, + * and fetches existing ProgramLocation objects based on the database IDs. It then creates a map of ProgramLocation objects by their names + * with PendingImportObject wrappers that indicate the state of the object as existing. + * + * @param program the Program object to associate with the locations + * @param studyByName a map of BrAPIStudy objects by their names + * @return a map of ProgramLocation objects by their names with PendingImportObject wrappers + * @throws InternalServerException if an error occurs during the location retrieval process + */ + Map> initializeLocationByName( + Program program, + Map> studyByName) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByName.size() > 0) { + Set locationDbIds = studyByName.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + existingLocations.forEach(existingLocation -> locationByName.put( + existingLocation.getName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) + ) + ); + return locationByName; + } private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { Map> locationByName = new HashMap<>(); @@ -1507,6 +1896,99 @@ private Map> initializeUniqueLocati existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); return locationByName; } + + /** + * Maps a given germplasm to an observation unit ID in a given map of germplasm by observation unit ID. + * + * This method retrieves the Global Identifier (GID) of the provided observation unit and uses it to lookup + * the corresponding PendingImportObject in the map of germplasm by name. The found germplasm + * object is then mapped to the observation unit ID in the provided map of germplasm by observation unit ID. + * The updated map is returned after the mapping operation has been performed. + * + * @param unitId The observation unit ID to which the germplasm should be mapped. + * @param unit The BrAPIObservationUnit object representing the observation unit. + * @param germplasmByName The map of germplasm objects by name used to lookup the desired germplasm. + * @param germplasmByOUId The map of germplasm objects by observation unit ID to update with the mapping result. + * @return The updated map of germplasm objects by observation unit ID after mapping the germplasm to the provided observation unit ID. + */ + private Map> mapGermplasmByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> germplasmByName, + Map> germplasmByOUId) { + String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); + germplasmByOUId.put(unitId, germplasmByName.get(gid)); + + return germplasmByOUId; + } + + /** + * Maps the pending observation dataset by OU Id based on the given inputs. + * This function checks if the trialByOUId map is not empty, the obsVarDatasetByName map is not empty, + * and if the first entry in the trialByOUId map contains observation dataset id in its additional info. + * If the conditions are met, it adds the pending import object from the obsVarDatasetByName map to the + * obsVarDatasetByOUId map using the unitId as the key. + * + * @param unitId the unit ID based on which the mapping is done + * @param trialByOUId a map containing pending import objects with BrAPITrial as the value, mapped by OU Id + * @param obsVarDatasetByName a map containing pending import objects with BrAPIListDetails as the value, mapped by dataset name + * @param obsVarDatasetByOUId a map containing pending import objects with BrAPIListDetails as the value, mapped by OU Id + * @return the updated obsVarDatasetByOUId map after potential addition of a pending import object + */ + private Map> mapPendingObsDatasetByOUId( + String unitId, + Map> trialByOUId, + Map> obsVarDatasetByName, + Map> obsVarDatasetByOUId) { + if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && + trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); + } + + return obsVarDatasetByOUId; + } + + /** + * Initializes observation variable dataset for existing observation units. This function retrieves existing datasets related to observation variables for the specified trial and program, processes the dataset details, and caches the data accordingly. + * + * @param trialByName A map containing trial information indexed by trial name. + * @param program The program to which the datasets are related. + * @return A map of observation variable dataset objects indexed by dataset name. + * @throws InternalServerException If the existing dataset summary is not retrieved from the BrAPI server, or an error occurs during API communication. + */ + private Map> initializeObsVarDatasetForExistingObservationUnits( + Map> trialByName, + Program program) { + Map> obsVarDatasetByName = new HashMap<>(); + + if (trialByName.size() > 0 && + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialByName.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + private Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { Map> obsVarDatasetByName = new HashMap<>(); @@ -1553,13 +2035,51 @@ private Optional> getTrialPIO(List> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); obsVarDatasetByName.put(existingList.getListName(), - new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceID()))); + new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } + + /** + * Initializes a mapping of BrAPI Germplasm objects by Germplasm ID for existing BrAPI Observation Units. + * This method retrieves existing Germplasms associated with the provided Observation Units and creates a mapping + * using their Accession Number as the key and a PendingImportObject containing the Germplasm object and a reference ID. + * If no existing Germplasms are found, an empty mapping is returned. + * + * @param unitByName A mapping of Observation Units by name. + * @param program The BrAPI Program object to which the Germplasms belong. + * @return A mapping of BrAPI Germplasm objects by Germplasm ID for existing Observation Units. + * @throws InternalServerException If an error occurs while fetching Germplasms from the database. + */ + private Map> initializeGermplasmByGIDForExistingObservationUnits( + Map> unitByName, + Program program) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(unitByName.size() > 0) { + Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; } + + private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); @@ -1590,7 +2110,7 @@ private Map> initializeExistingGermp existingGermplasms.forEach(existingGermplasm -> { BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceID()))); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); }); return existingGermplasmByGID; } @@ -1600,15 +2120,19 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID()); + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); observationUnitByName.put(createObservationUnitKey(row), new PendingImportObject<>(ImportObjectState.EXISTING, brAPIObservationUnit, - UUID.fromString(idRef.getReferenceID()))); + UUID.fromString(idRef.getReferenceId()))); } - - private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { + private PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); // map season dbid to year @@ -1617,9 +2141,16 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map String seasonYear = this.seasonDbIdToYear(seasonDbId, program.getId()); existingStudy.setSeasons(Collections.singletonList(seasonYear)); } - studyByName.put( - Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), UUID.fromString(xref.getReferenceID()))); + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; } private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { @@ -1628,18 +2159,18 @@ private void initializeTrialsForExistingObservationUnits(Program program, Map studyDbIds = new HashSet<>(); observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId if(!studyDbIds.isEmpty()) { @@ -1659,8 +2190,7 @@ private void initializeTrialsForExistingObservationUnits(Program program, Map processAndCacheTrial(trial, program, trialRefSource, trialByName)); + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1681,18 +2211,51 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr return trialDbIds; } - private void processAndCacheTrial(BrAPITrial existingTrial, Program program, String trialRefSource, Map> trialByNameNoScope) { + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource) + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceID()); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); trialByNameNoScope.put( Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } + /** + * This function collates unique ObsUnitID values from a list of BrAPIImport objects. + * It iterates through the list and adds non-blank ObsUnitID values to a Set. + * It also checks for any repeated ObsUnitIDs. The instance variables hasAllReferenceUnitIds and + * hasNoReferenceUnitIds are updated. + * + * @param importRows a List of BrAPIImport objects containing ExperimentObservation data + * @return a Set of unique ObsUnitID strings + * @throws IllegalStateException if a repeated ObsUnitID is encountered + */ + private Set collateReferenceOUIds(List importRows) { + Set referenceOUIds = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + return referenceOUIds; + } + private void validateTimeStampValue(String value, String columnHeader, ValidationErrors validationErrors, int row) { if (StringUtils.isBlank(value)) { @@ -1857,4 +2420,19 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { Integer yearInt = (season == null) ? null : season.getYear(); return (yearInt == null) ? "" : yearInt.toString(); } + + /** + * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. + * + * @param map The map from which to retrieve the single value. + * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. + * @return The single value from the map. + * @throws UnprocessableEntityException if the map does not contain exactly one entry. + */ + private V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { + if (map.size() != 1) { + throw new UnprocessableEntityException(message); + } + return map.values().iterator().next(); + } } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 7a797a815..6bc203d64 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -361,7 +361,6 @@ public void importNewEnvNoObsSuccess() { assertRowSaved(newEnv, program, null); } - @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows @@ -599,50 +598,49 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); } - // NO Longer needed, but may be needed in the future. -// @ParameterizedTest -// @ValueSource(booleans = {true, false}) -// @SneakyThrows -// public void verifyFailureNewOuExistingEnv(boolean commit) { -// Program program = createProgram("New OU Exising Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); -// Map newExp = new HashMap<>(); -// newExp.put(Columns.GERMPLASM_GID, "1"); -// newExp.put(Columns.TEST_CHECK, "T"); -// newExp.put(Columns.EXP_TITLE, "Test Exp"); -// newExp.put(Columns.EXP_UNIT, "Plot"); -// newExp.put(Columns.EXP_TYPE, "Phenotyping"); -// newExp.put(Columns.ENV, "New Env"); -// newExp.put(Columns.ENV_LOCATION, "Location A"); -// newExp.put(Columns.ENV_YEAR, "2023"); -// newExp.put(Columns.EXP_UNIT_ID, "a-1"); -// newExp.put(Columns.REP_NUM, "1"); -// newExp.put(Columns.BLOCK_NUM, "1"); -// newExp.put(Columns.ROW, "1"); -// newExp.put(Columns.COLUMN, "1"); -// -// importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); -// -// Map newOU = new HashMap<>(newExp); -// newOU.put(Columns.EXP_UNIT_ID, "a-2"); -// newOU.put(Columns.ROW, "1"); -// newOU.put(Columns.COLUMN, "2"); -// -// Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); -// HttpResponse response = call.blockingFirst(); -// assertEquals(HttpStatus.ACCEPTED, response.getStatus()); -// -// String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); -// -// HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); -// JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); -// assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); -// -// assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); -// } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyFailureNewOuExistingEnv(boolean commit) { + Program program = createProgram("New OU Existing Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + Map newOU = new HashMap<>(newExp); + newOU.put(Columns.EXP_UNIT_ID, "a-2"); + newOU.put(Columns.ROW, "1"); + newOU.put(Columns.COLUMN, "2"); + + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + } @Test @SneakyThrows - public void importNewObsVarExisingOu() { + public void importNewObsVarExistingOu() { List traits = importTestUtils.createTraits(2); Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -666,7 +664,7 @@ public void importNewObsVarExisingOu() { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -686,7 +684,7 @@ public void importNewObsVarExisingOu() { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); @@ -704,11 +702,131 @@ public void importNewObsVarExisingOu() { assertRowSaved(newObsVar, program, traits); } + @Test + @SneakyThrows + public void importNewObsVarByObsUnitId() { + List traits = importTestUtils.createTraits(2); + Program program = createProgram("New ObsVar Referring to OU by ID", "OUVAR", "VAROU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put(traits.get(1).getObservationVariableName(), null); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowReferencedByOUIdSaved(newObsVar, program, traits); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importNewObservationDataByObsUnitId(boolean commit) { + List traits = importTestUtils.createTraits(1); + Program program = createProgram("New Observation Referring to OU by ID"+(commit ? "C" : "P"), "OUDAT"+(commit ? "C" : "P"), "DATOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + if(commit) { + assertRowSaved(newObsVar, program, traits); + } else { + assertValidPreviewRow(newObsVar, row, program, traits); + } + } + @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows - public void importNewObsExisingOu(boolean commit) { + public void importNewObsExistingOu(boolean commit) { List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing OU "+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -731,7 +849,7 @@ public void importNewObsExisingOu(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -751,7 +869,7 @@ public void importNewObsExisingOu(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); @@ -774,7 +892,7 @@ public void importNewObsExisingOu(boolean commit) { @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows - public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { + public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing Obs "+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -798,7 +916,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -818,7 +936,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); @@ -925,7 +1043,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -945,7 +1063,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -999,7 +1117,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -1021,7 +1139,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -1036,7 +1154,6 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - newObservation.put(traits.get(0).getObservationVariableName(), "1"); if(commit) { assertRowSaved(newObservation, program, traits); } else { @@ -1044,6 +1161,41 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { } } + private Map assertRowReferencedByOUIdSaved(Map expected, Program program, List traits) throws ApiException { + Map ret = new HashMap<>(); + + List units = ouDAO.getObservationUnitsById(List.of((String)expected.get(Columns.OBS_UNIT_ID)), program); + assertFalse(units.isEmpty()); + + List observations = null; + if(traits != null) { + observations = observationDAO.getObservationsByStudyName(List.of(units.get(0).getStudyName()), program); + if (expected.get(traits.get(0).getObservationVariableName()) == null) { + assertTrue(observations.isEmpty()); + } else { + assertFalse(observations.isEmpty()); + List expectedVariableObservation = new ArrayList<>(); + List actualVariableObservation = new ArrayList<>(); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + for(Trait trait : traits) { + if (expected.get(trait.getObservationVariableName()) != null) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + } + if (actualVariableObservation.isEmpty()) { + assertTrue(expectedVariableObservation.isEmpty()); + } else { + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); + } + + } + + ret.put("observations", observations); + } + + return ret; + } + private Map assertRowSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); @@ -1053,7 +1205,7 @@ private Map assertRowSaved(Map expected, Program Optional trialIdXref = Utilities.getExternalReference(trial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program); + List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program); assertFalse(studies.isEmpty()); BrAPIStudy study = null; for(BrAPIStudy s : studies) { @@ -1147,7 +1299,14 @@ private Map assertRowSaved(Map expected, Program if(traits != null) { List expectedVariableObservation = new ArrayList<>(); List actualVariableObservation = new ArrayList<>(); - observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + observations.forEach(observation -> actualVariableObservation.add( + String.format( + "%s:%s", + Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), + observation.getValue() + ) + )); + for(Trait trait : traits) { if (expected.get(trait.getObservationVariableName()) != null) { expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName())));