Skip to content

BI-1778 - Environments locations and years #250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
timparsons merged 1 commit into from
May 11, 2023
Merged

BI-1778 - Environments locations and years #250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.micronaut.test</groupId>
<artifactId>micronaut-test-junit5</artifactId>
Expand Down
102 changes: 78 additions & 24 deletions ...ain/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ public class ExperimentProcessor implements Processor {
private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment";
private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment";
private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations";
private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment";
private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment";

@Property(name = "brapi.server.reference-source")
private String BRAPI_REFERENCE_SOURCE;
Expand Down Expand Up @@ -228,7 +230,7 @@ public Map<String, ImportPreviewStatistics> process(

prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport);

validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols);
validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit);

if (validationErrors.hasErrors()) {
throw new ValidatorException(validationErrors);
Expand Down Expand Up @@ -540,7 +542,7 @@ private String getObservationHash(String observationUnitName, String variableNam
}

private ValidationErrors validateFields(List<BrAPIImport> importRows, ValidationErrors validationErrors, Map<Integer, PendingImport> mappedBrAPIImport, List<Trait> referencedTraits, Program program,
List<Column<?>> phenotypeCols) throws MissingRequiredInfoException, ApiException {
List<Column<?>> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException {
//fetching any existing observations for any OUs in the import
Map<String, BrAPIObservation> existingObsByObsHash = fetchExistingObservations(referencedTraits, program);
Map<String, Trait> colVarMap = referencedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity()));
Expand All @@ -560,7 +562,7 @@ private ValidationErrors validateFields(List<BrAPIImport> importRows, Validation
throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR);
}

validateConditionallyRequired(validationErrors, rowNum, importRow);
validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit);
validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow);
validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash);
}
Expand Down Expand Up @@ -659,7 +661,7 @@ private void validateUniqueObsUnits(
}
}

private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) {
private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) {
ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle())
.getState();
ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState();
Expand All @@ -677,8 +679,21 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in
validateRequiredCell(importRow.getExpUnit(), Columns.EXP_UNIT, errorMessage, validationErrors, rowNum);
validateRequiredCell(importRow.getExpType(), Columns.EXP_TYPE, errorMessage, validationErrors, rowNum);
validateRequiredCell(importRow.getEnv(), Columns.ENV, errorMessage, validationErrors, rowNum);
validateRequiredCell(importRow.getEnvLocation(), Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum);
validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum);
if(validateRequiredCell(importRow.getEnvLocation(), Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) {
if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) {
addRowError(Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum);
}
}
if(validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) {
String studyYear = this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0);
String rowYear = importRow.getEnvYear();
if(commit) {
rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId());
}
if(!studyYear.equals(rowYear)) {
addRowError(Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum);
}
}
validateRequiredCell(importRow.getExpUnitId(), Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum);
validateRequiredCell(importRow.getExpReplicateNo(), Columns.REP_NUM, errorMessage, validationErrors, rowNum);
validateRequiredCell(importRow.getExpBlockNo(), Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum);
Expand All @@ -691,10 +706,12 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in
}
}

private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) {
private boolean validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) {
if (StringUtils.isBlank(value)) {
addRowError(columnHeader, errorMessage, validationErrors, rowNum);
return false;
}
return true;
}

private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) {
Expand Down Expand Up @@ -1091,15 +1108,20 @@ private Map<String, PendingImportObject<BrAPIObservationUnit>> initializeObserva
String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName());
if (existingObsUnits.size() == rowByObsUnitId.size()) {
existingObsUnits.forEach(brAPIObservationUnit -> {
processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, ret, rowByObsUnitId);

BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource)
.orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references"));

ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID());
row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey()));
ret.put(createObservationUnitKey(row),
new PendingImportObject<>(ImportObjectState.EXISTING,
brAPIObservationUnit,
UUID.fromString(idRef.getReferenceID())));
row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey()));
row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey()));
row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey()));
});
} else {
List<String> missingIds = new ArrayList<>(rowByObsUnitId.keySet());
missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()));
throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds));
}

return ret;
Expand Down Expand Up @@ -1138,6 +1160,13 @@ private Map<String, PendingImportObject<BrAPIStudy>> initializeStudyByNameNoScop
return studyByName;
}

try {
initializeStudiesForExistingObservationUnits(program, studyByName);
} catch (ApiException e) {
log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e);
throw new InternalServerException(e.toString(), e);
}

List<BrAPIStudy> existingStudies;
Optional<PendingImportObject<BrAPITrial>> trial = getTrialPIO(experimentImportRows);

Expand All @@ -1156,6 +1185,29 @@ private Map<String, PendingImportObject<BrAPIStudy>> initializeStudyByNameNoScop
return studyByName;
}

private void initializeStudiesForExistingObservationUnits(Program program, Map<String, PendingImportObject<BrAPIStudy>> studyByName) throws ApiException {
Set<String> studyDbIds = this.observationUnitByNameNoScope.values()
.stream()
.map(pio -> pio.getBrAPIObject()
.getStudyDbId())
.collect(Collectors.toSet());

List<BrAPIStudy> studies = fetchStudiesByDbId(studyDbIds, program);
studies.forEach(study -> {
processAndCacheStudy(study, program, studyByName);
});
}

private List<BrAPIStudy> fetchStudiesByDbId(Set<String> studyDbIds, Program program) throws ApiException {
List<BrAPIStudy> studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program);
if(studies.size() != studyDbIds.size()) {
List<String> missingIds = new ArrayList<>(studyDbIds);
missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList()));
throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds));
}
return studies;
}

private Map<String, PendingImportObject<ProgramLocation>> initializeUniqueLocationNames(Program program, List<ExperimentObservation> experimentImportRows) {
Map<String, PendingImportObject<ProgramLocation>> locationByName = new HashMap<>();

Expand Down Expand Up @@ -1279,17 +1331,24 @@ private Map<String, PendingImportObject<BrAPIGermplasm>> initializeExistingGermp
return existingGermplasmByGID;
}

private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map<String, PendingImportObject<BrAPIStudy>> studyByName) {
//Swap season DbId with year String
String seasonId = existingStudy.getSeasons().get(0);
existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId())));
private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, Map<String, PendingImportObject<BrAPIObservationUnit>> observationUnitByName,
Map<String, ExperimentObservation> rowByObsUnitId) {
BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource)
.orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references"));

existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()));
ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID());
row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey()));
observationUnitByName.put(createObservationUnitKey(row),
new PendingImportObject<>(ImportObjectState.EXISTING,
brAPIObservationUnit,
UUID.fromString(idRef.getReferenceID())));
}

private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map<String, PendingImportObject<BrAPIStudy>> studyByName) {
BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName()))
.orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId()));
studyByName.put(
existingStudy.getStudyName(),
Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()),
new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.getReferenceID())));
}

Expand Down Expand Up @@ -1341,12 +1400,7 @@ private void initializeTrialsForExistingObservationUnits(Program program, Map<St

private Set<String> fetchTrialDbidsForStudies(Set<String> studyDbIds, Program program) throws ApiException {
Set<String> trialDbIds = new HashSet<>();
List<BrAPIStudy> studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program);
if(studies.size() != studyDbIds.size()) {
List<String> missingIds = new ArrayList<>(trialDbIds);
missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList()));
throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds));
}
List<BrAPIStudy> studies = fetchStudiesByDbId(studyDbIds, program);
studies.forEach(study -> {
if (StringUtils.isBlank(study.getTrialDbId())) {
throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId());
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/breedinginsight/utilities/Utilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,13 @@ public static String appendProgramKey( String original, String programKey ){
* @return
*/
public static String removeProgramKey(String original, String programKey, String additionalKeyData) {
String keyValue;
if(StringUtils.isNotBlank(additionalKeyData)) {
String keyValue = String.format(" [%s-%s]", programKey, additionalKeyData);
return original.replace(keyValue, "");
keyValue = String.format(" [%s-%s]", programKey, additionalKeyData);
} else {
String keyValue = String.format(" [%s]", programKey);
return original.replace(keyValue, "");
keyValue = String.format(" [%s]", programKey);
}
return original.replace(keyValue, "");
}

/**
Expand Down
Loading