From 6804c88ff47d0758771ccbbc59953286a2788ea9 Mon Sep 17 00:00:00 2001 From: timparsons Date: Sat, 21 Jan 2023 17:40:47 -0500 Subject: [PATCH 01/15] [BI-1195] Adding functionality to fetch existing observation units on experiment file import --- .../daos/BrAPIObservationUnitDAO.java | 23 +- .../importer/services/FileImportService.java | 12 +- .../processors/ExperimentProcessor.java | 739 ++++++++++-------- .../breedinginsight/utilities/Utilities.java | 8 + 4 files changed, 454 insertions(+), 328 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index f9a313455..39aaa3a11 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -17,11 +17,13 @@ package org.breedinginsight.brapps.importer.daos; +import io.micronaut.context.annotation.Property; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -29,8 +31,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.UUID; +import java.util.*; @Singleton public class BrAPIObservationUnitDAO { @@ -42,12 +43,15 @@ public class BrAPIObservationUnitDAO { private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final String referenceSource; + @Inject - public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, @Property(name = "brapi.server.reference-source") String referenceSource) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.referenceSource = referenceSource; } /* @@ -85,4 +89,17 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); + observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() + .getProgramDbId())); + observationUnitSearchRequest.externalReferenceIDs(new ArrayList<>(observationUnitExternalIds)); + observationUnitSearchRequest.externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))); + + ObservationUnitsApi api = new ObservationUnitsApi(programDAO.getCoreClient(program.getId())); + return brAPIDAOUtil.search(api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + observationUnitSearchRequest); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 3bd85b6df..8f7f3f549 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -329,7 +329,9 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU // Find the import Optional uploadOptional = importDAO.getUploadById(uploadId); - if (uploadOptional.isEmpty()) throw new DoesNotExistException("Upload with that id does not exist"); + if (uploadOptional.isEmpty()) { + throw new DoesNotExistException("Upload with that id does not exist"); + } ImportUpload upload = uploadOptional.get(); if (upload.getProgress() != null && upload.getProgress().getStatuscode().equals((short) HttpStatus.ACCEPTED.getCode())) { @@ -344,11 +346,15 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU // Get mapping Optional mappingConfigOptional = importMappingDAO.getMapping(upload.getImporterMappingId()); - if (mappingConfigOptional.isEmpty()) throw new DoesNotExistException("Cannot find mapping config associated with upload."); + if (mappingConfigOptional.isEmpty()) { + throw new DoesNotExistException("Cannot find mapping config associated with upload."); + } ImportMapping mappingConfig = mappingConfigOptional.get(); Optional optionalImportService = configManager.getImportServiceById(mappingConfig.getImportTypeId()); - if (optionalImportService.isEmpty()) throw new DoesNotExistException("Config with that id does not exist"); + if (optionalImportService.isEmpty()) { + throw new DoesNotExistException("Config with that id does not exist"); + } BrAPIImportService importService = optionalImportService.get(); // TODO: maybe return brapiimport from configmanager diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index dda9a417e..414a503e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -31,16 +31,18 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -98,9 +100,9 @@ public class ExperimentProcessor implements Processor { private FileMappingUtil fileMappingUtil; // used to make the yearsToSeasonDbId() function more efficient - private final Map yearToSeasonDbIdCache = new HashMap<>(); + private final Map yearToSeasonDbIdCache = new HashMap<>(); // used to make the seasonDbIdtoYear() function more efficient - private final Map seasonDbIdToYearCache = new HashMap<>(); + private final Map seasonDbIdToYearCache = new HashMap<>(); //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. @@ -144,6 +146,7 @@ public ExperimentProcessor(DSLContext dsl, /** * Initialize the Map objects with existing BrAPI Data. + * * @param importRows * @param program */ @@ -151,26 +154,24 @@ public ExperimentProcessor(DSLContext dsl, public void getExistingBrapiData(List importRows, Program program) { List experimentImportRows = importRows.stream() - .map(trialImport -> (ExperimentObservation) trialImport) - .collect(Collectors.toList()); - - this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); - this.locationByName = initialize_uniqueLocationNames( program, experimentImportRows ); - this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); - // All of the Observation Units will be new. None will be preexisting. - this.observationUnitByNameNoScope = new HashMap<>(); - // TODO: populate existing observations, assume all new currently + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + + this.trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); + this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); + this.studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); + this.observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); // key and removing key - this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); + this.existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); } /** - * @param importRows - one element of the list for every row of the import file. + * @param importRows - one element of the list for every row of the import file. * @param mappedBrAPIImport - passed in by reference and modified within this program (this will later be passed to the front end for the preview) * @param program * @param user - * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) - * false when used for preview only + * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * false when used for preview only * @return Map - used to display the summary statistics. * @throws ValidatorException */ @@ -189,7 +190,7 @@ public Map process( List> dynamicCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME); List> phenotypeCols = new ArrayList<>(); List> timestampCols = new ArrayList<>(); - for (Column dynamicCol: dynamicCols) { + for (Column dynamicCol : dynamicCols) { //Distinguish between phenotype and timestamp columns if (dynamicCol.name().startsWith("TS:")) { timestampCols.add(dynamicCol); @@ -198,61 +199,12 @@ public Map process( } } - List varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toList()); - List tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toList()); - - // Lookup all traits in system for program, maybe eventually add a variable search in ontology service - List traits = getTraitList(program); - - // filter out just traits specified in file - List filteredTraits = traits.stream() - .filter(e -> varNames.contains(e.getObservationVariableName())) - .collect(Collectors.toList()); - - // check that all specified ontology terms were found - if (filteredTraits.size() != varNames.size()) { - List returnedVarNames = filteredTraits.stream().map(TraitEntity::getObservationVariableName) - .collect(Collectors.toList()); - List differences = varNames.stream() - .filter(var -> !returnedVarNames.contains(var)) - .collect(Collectors.toList()); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Ontology term(s) not found: " + String.join(", ", differences)); - } - - // Check that each ts column corresponds to a phenotype column - List unmatchedTimestamps = tsNames.stream() - .filter(e -> !(varNames.contains(e.replaceFirst("^TS:\\s*","")))) - .collect(Collectors.toList()); - if (unmatchedTimestamps.size() > 0) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(", ", unmatchedTimestamps)); - } + validateObservationAndTimestampColumns(program.getId(), phenotypeCols, timestampCols, validationErrors); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - for (Column tsColumn: timestampCols) { - timeStampColByPheno.put(tsColumn.name().replaceFirst("^TS:\\s*",""), tsColumn); - } - - // Perform ontology validations on each observation value in phenotype column - Map colVarMap = filteredTraits.stream() - .collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); - - for (Column column : phenotypeCols) { - for (int i=0; i < column.size(); i++) { - String value = column.getString(i); - String colName = column.name(); - validateObservationValue(colVarMap.get(colName), value, colName, validationErrors, i); - } - } - - //Timestamp validation - for (Column column : timestampCols) { - for (int i=0; i < column.size(); i++) { - String value = column.getString(i); - String colName = column.name(); - validateTimeStampValue(value, colName, validationErrors, i); - } + for (Column tsColumn : timestampCols) { + timeStampColByPheno.put(tsColumn.name() + .replaceFirst("^TS:\\s*", ""), tsColumn); } // add "New" pending data to the BrapiData objects @@ -263,10 +215,10 @@ public Map process( ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - mappedImportRow.setTrial( this.trialByNameNoScope.get( importRow.getExpTitle() ) ); - mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); - mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); - mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ); + mappedImportRow.setTrial(this.trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(this.locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(this.studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(this.observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); // loop over phenotype column observation data for current row for (Column column : phenotypeCols) { @@ -277,14 +229,15 @@ public Map process( } PendingImportObject germplasmPIO = getGidPOI(importRow); - mappedImportRow.setGermplasm( germplasmPIO ); + mappedImportRow.setGermplasm(germplasmPIO); - if (! StringUtils.isBlank( importRow.getGid() )) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow,validationErrors, i, germplasmPIO); + if (!StringUtils.isBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, i, germplasmPIO); } //Check if existing environment. If so, ObsUnitId must be assigned - if ((this.studyByNameNoScope.get(importRow.getEnv()).getState() == ImportObjectState.EXISTING) && (StringUtils.isBlank(importRow.getObsUnitID()))){ + if ((this.studyByNameNoScope.get(importRow.getEnv()) + .getState() == ImportObjectState.EXISTING) && (StringUtils.isBlank(importRow.getObsUnitID()))) { throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); } @@ -294,7 +247,7 @@ public Map process( validationErrors = validateFields(importRows, validationErrors); - if (validationErrors.hasErrors()){ + if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } @@ -302,6 +255,74 @@ public Map process( return getStatisticsMap(importRows); } + private void validateObservationAndTimestampColumns(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { + Set varNames = phenotypeCols.stream() + .map(Column::name) + .collect(Collectors.toSet()); + Set tsNames = timestampCols.stream() + .map(Column::name) + .collect(Collectors.toSet()); + + // filter out just traits specified in file + List filteredTraits = fetchFileTraits(programId, varNames); + + // check that all specified ontology terms were found + if (filteredTraits.size() != varNames.size()) { + Set returnedVarNames = filteredTraits.stream() + .map(TraitEntity::getObservationVariableName) + .collect(Collectors.toSet()); + List differences = varNames.stream() + .filter(var -> !returnedVarNames.contains(var)) + .collect(Collectors.toList()); + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Ontology term(s) not found: " + String.join(", ", differences)); + } + + // Check that each ts column corresponds to a phenotype column + List unmatchedTimestamps = tsNames.stream() + .filter(e -> !(varNames.contains(e.replaceFirst("^TS:\\s*", "")))) + .collect(Collectors.toList()); + if (unmatchedTimestamps.size() > 0) { + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(", ", unmatchedTimestamps)); + } + + // Perform ontology validations on each observation value in phenotype column + Map colVarMap = filteredTraits.stream() + .collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); + + for (Column column : phenotypeCols) { + for (int i = 0; i < column.size(); i++) { + String value = column.getString(i); + String colName = column.name(); + validateObservationValue(colVarMap.get(colName), value, colName, validationErrors, i); + } + } + + //Timestamp validation + for (Column column : timestampCols) { + for (int i = 0; i < column.size(); i++) { + String value = column.getString(i); + String colName = column.name(); + validateTimeStampValue(value, colName, validationErrors, i); + } + } + } + + private List fetchFileTraits(UUID programId, Collection varNames) { + try { + List traits = ontologyService.getTraitsByProgramId(programId, true); + // filter out just traits specified in file + return traits.stream() + .filter(e -> varNames.contains(e.getObservationVariableName())) + .collect(Collectors.toList()); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + throw new InternalServerException(e.toString(), e); + } + } private String getVariableNameFromColumn(Column column) { // TODO: timestamp stripping? @@ -324,7 +345,7 @@ private void getNewBrapiData(List importRows, List> pheno } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - for (int i=0; i importRows, List> pheno this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); String expSeqValue = null; - if(commit) { - expSeqValue = trialPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(); + if (commit) { + expSeqValue = trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString(); } PendingImportObject locationPIO = createLocationPIO(importRow); @@ -343,8 +367,11 @@ private void getNewBrapiData(List importRows, List> pheno this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); String envSeqValue = null; - if(commit) { - envSeqValue = studyPIO.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + if (commit) { + envSeqValue = studyPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER) + .getAsString(); } PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow); @@ -355,10 +382,11 @@ private void getNewBrapiData(List importRows, List> pheno //If associated timestamp column, add String dateTimeValue = null; if (timeStampColByPheno.get(column.name()) != null) { - dateTimeValue = timeStampColByPheno.get(column.name()).getString(i); + dateTimeValue = timeStampColByPheno.get(column.name()) + .getString(i); //If no timestamp, set to midnight - if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)){ - dateTimeValue+="T00:00:00-00:00"; + if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { + dateTimeValue += "T00:00:00-00:00"; } } //column.name() gets phenotype name @@ -370,7 +398,7 @@ private void getNewBrapiData(List importRows, List> pheno } private String createObservationUnitKey(ExperimentObservation importRow) { - String key = createObservationUnitKey( importRow.getEnv(), importRow.getExpUnitId() ); + String key = createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); return key; } @@ -404,79 +432,80 @@ private ValidationErrors validateFields(List importRows, Validation * Validate that the the observation unit is unique within a study. *
* SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. - * @param validationErrors can be modified as a side effect. + * + * @param validationErrors can be modified as a side effect. * @param uniqueStudyAndObsUnit can be modified as a side effect. - * @param i counter that is always two less the file row being validated - * @param importRow the data row being validated + * @param i counter that is always two less the file row being validated + * @param importRow the data row being validated */ private void validateUniqueObsUnits( ValidationErrors validationErrors, HashSet uniqueStudyAndObsUnit, int i, ExperimentObservation importRow) { - String envIdPlusStudyId = createObservationUnitKey( importRow ); - if( uniqueStudyAndObsUnit.contains( envIdPlusStudyId )){ + String envIdPlusStudyId = createObservationUnitKey(importRow); + if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); - } - else{ - uniqueStudyAndObsUnit.add( envIdPlusStudyId ); + } else { + uniqueStudyAndObsUnit.add(envIdPlusStudyId); } } private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { String experimentTitle = importRow.getExpTitle(); String obsUnitID = importRow.getObsUnitID(); - if( StringUtils.isBlank( obsUnitID )){ + if (StringUtils.isBlank(obsUnitID)) { validateRequiredCell( experimentTitle, "Exp Title", "Field is blank", validationErrors, i - ); + ); } - ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); + ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle) + .getState(); boolean isExperimentNew = (expState == ImportObjectState.NEW); if (isExperimentNew) { String errorMessage = "Field is blank when creating a new experiment"; - validateRequiredCell( importRow.getGid(), - "GID", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpUnit(), - "Exp Unit", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpType(), - "Exp Type", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getEnv(), - "Env", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getEnvLocation(), - "Env Location", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getEnvYear(), - "Env Year", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpUnitId(), - "Exp Unit ID", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpReplicateNo(), - "Exp Replicate #", - errorMessage, validationErrors, i); - validateRequiredCell( importRow.getExpBlockNo(), - "Exp Block #", - errorMessage, validationErrors, i); + validateRequiredCell(importRow.getGid(), + "GID", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getExpUnit(), + "Exp Unit", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getExpType(), + "Exp Type", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getEnv(), + "Env", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getEnvLocation(), + "Env Location", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getEnvYear(), + "Env Year", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getExpUnitId(), + "Exp Unit ID", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getExpReplicateNo(), + "Exp Replicate #", + errorMessage, validationErrors, i); + validateRequiredCell(importRow.getExpBlockNo(), + "Exp Block #", + errorMessage, validationErrors, i); } } private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int i) { - if ( StringUtils.isBlank( value )) { + if (StringUtils.isBlank(value)) { addRowError( columnHeader, errorMessage, validationErrors, i - ) ; + ); } } @@ -486,7 +515,9 @@ private void addRowError(String field, String errorMessage, ValidationErrors val } private void addIfNotNull(HashSet set, String setValue) { - if( setValue!=null) { set.add(setValue); } + if (setValue != null) { + set.add(setValue); + } } private Map getStatisticsMap(List importRows) { @@ -499,41 +530,43 @@ private Map getStatisticsMap(List ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, createObservationUnitKey( importRow )); + addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); addIfNotNull(gidCounter, importRow.getGid()); } int numNewObservations = Math.toIntExact( - observationByHash.values().stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && - !StringUtils.isBlank(preview.getBrAPIObject().getValue())) - .count() + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() ); ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameCounter.size()) - .build(); + .newObjectCount(environmentNameCounter.size()) + .build(); ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDCounter.size()) - .build(); + .newObjectCount(obsUnitsIDCounter.size()) + .build(); ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(gidCounter.size()) - .build(); + .newObjectCount(gidCounter.size()) + .build(); ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(numNewObservations) - .build(); + .newObjectCount(numNewObservations) + .build(); return Map.of( - "Environments", environmentStats, - "Observation_Units", obdUnitStats, - "GIDs", gidStats, - "Observations", observationStats + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats ); } private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { - // error if GID is not blank but GID does not already exist - if( !StringUtils.isBlank( importRow.getGid()) && germplasmPIO == null ) { + // error if GID is not blank but GID does not already exist + if (!StringUtils.isBlank(importRow.getGid()) && germplasmPIO == null) { addRowError( "GID", "A non-existing GID", @@ -543,23 +576,23 @@ private void validateGermplasm(ExperimentObservation importRow, ValidationErrors } private PendingImportObject getGidPOI(ExperimentObservation importRow) { - if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ + if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { return existingGermplasmByGID.get(importRow.getGid()); - } - else{ + } else { return null; } } private PendingImportObject createObsUnitPIO(Program program, boolean commit, String seqValue, ExperimentObservation importRow) { PendingImportObject pio = null; - if( this.observationUnitByNameNoScope.containsKey( createObservationUnitKey( importRow ) ) ) { - pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; - } - else{ + if (this.observationUnitByNameNoScope.containsKey(createObservationUnitKey(importRow))) { + pio = observationUnitByNameNoScope.get(createObservationUnitKey(importRow)); + } else { String germplasmName = ""; - if( this.existingGermplasmByGID.get( importRow.getGid() ) != null) { - germplasmName = this.existingGermplasmByGID.get(importRow.getGid()).getBrAPIObject().getGermplasmName(); + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); } PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); @@ -573,12 +606,17 @@ private PendingImportObject createObsUnitPIO(Program progr } - private PendingImportObject createObservationPIO(ExperimentObservation importRow, String variableName, String value, String timeStampValue, boolean commit, String seasonDbId, PendingImportObject obsUnitPIO) { + private PendingImportObject createObservationPIO(ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO) { PendingImportObject pio = null; if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { pio = observationByHash.get(getImportObservationHash(importRow, variableName)); - } - else { + } else { BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject()); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream @@ -593,16 +631,15 @@ private PendingImportObject createObservationPIO(ExperimentObs private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { PendingImportObject pio = null; - if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { - pio = studyByNameNoScope.get( importRow.getEnv() ) ; - } - else{ + if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + } else { PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); - if( commit) { + if (commit) { String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season String seasonID = this.yearToSeasonDbId(year, program.getId()); newStudy.setSeasons(Arrays.asList(seasonID)); @@ -615,10 +652,9 @@ private PendingImportObject createStudyPIO(Program program, boolean private PendingImportObject createLocationPIO(ExperimentObservation importRow) { PendingImportObject pio = null; - if( locationByName.containsKey(( importRow.getEnvLocation() ))){ - pio = locationByName.get( importRow.getEnvLocation() ); - } - else{ + if (locationByName.containsKey((importRow.getEnvLocation()))) { + pio = locationByName.get(importRow.getEnvLocation()); + } else { BrAPILocation newLocation = importRow.constructBrAPILocation(); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation); } @@ -627,14 +663,14 @@ private PendingImportObject createLocationPIO(ExperimentObservati private PendingImportObject createTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { PendingImportObject pio = null; - if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { - pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; - } - else { + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + pio = trialByNameNoScope.get(importRow.getExpTitle()); + } else { UUID id = UUID.randomUUID(); String expSeqValue = null; - if(commit){ - expSeqValue = expNextVal.get().toString(); + if (commit) { + expSeqValue = expNextVal.get() + .toString(); } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); @@ -655,24 +691,25 @@ public void postBrapiData(Map mappedBrAPIImport, Program List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash).stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial - for ( BrAPITrial createdTrial: createdTrials ) { + for (BrAPITrial createdTrial : createdTrials) { String createdTrialName = createdTrial.getTrialName(); - String createdTrialName_no_key = Utilities.removeProgramKey( createdTrialName, program.getKey() ); + String createdTrialName_no_key = Utilities.removeProgramKey(createdTrialName, program.getKey()); PendingImportObject pi = this.trialByNameNoScope.get(createdTrialName_no_key); BrAPITrial listedTrial = pi.getBrAPIObject(); String dbid = createdTrial.getTrialDbId(); - listedTrial.setTrialDbId( dbid ); + listedTrial.setTrialDbId(dbid); } List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload)); // set the DbId to the for each newly created trial - for ( BrAPILocation createdLocation : createdLocations){ + for (BrAPILocation createdLocation : createdLocations) { String createdLocationName = createdLocation.getLocationName(); PendingImportObject pi = this.locationByName.get(createdLocationName); BrAPILocation listedLocation = pi.getBrAPIObject(); @@ -680,33 +717,33 @@ public void postBrapiData(Map mappedBrAPIImport, Program listedLocation.setLocationDbId(dbid); } - updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); List createdStudies = new ArrayList<>(); - createdStudies.addAll( brAPIStudyDAO.createBrAPIStudy(newStudies, program.getId(), upload) ); + createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(newStudies, program.getId(), upload)); // set the DbId to the for each newly created study - for( BrAPIStudy createdStudy : createdStudies){ - String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData( createdStudy.getStudyName(), program.getKey() ); - PendingImportObject pi = this.studyByNameNoScope.get( createdStudy_name_no_key ); + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + PendingImportObject pi = this.studyByNameNoScope.get(createdStudy_name_no_key); BrAPIStudy brAPIStudy = pi.getBrAPIObject(); - brAPIStudy.setStudyDbId( createdStudy.getStudyDbId() ); + brAPIStudy.setStudyDbId(createdStudy.getStudyDbId()); } updateObsUnitDependencyValues(program.getKey()); List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); // set the DbId to the for each newly created Observation Unit - for( BrAPIObservationUnit createdObservationUnit : createdObservationUnits){ + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope - String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getStudyName(),program.getKey() ); - String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( createdObservationUnit.getObservationUnitName(),program.getKey() ); + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); PendingImportObject pi = this.observationUnitByNameNoScope.get(createdObsUnit_key); // update the retrieved BrAPI object BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject(); - brAPIObservationUnit.setObservationUnitDbId ( createdObservationUnit.getObservationUnitDbId() ); + brAPIObservationUnit.setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); } - + updateObservationDependencyValues(program); brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload); } catch (ApiException e) { @@ -720,20 +757,20 @@ private void updateObservationDependencyValues(Program program) { // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds this.observationUnitByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); // Update ObservationVariable DbIds List traits = getTraitList(program); Map traitMap = traits.stream().collect(Collectors.toMap(trait -> trait.getObservationVariableName(), trait -> trait)); - for(PendingImportObject observation: this.observationByHash.values()){ + for (PendingImportObject observation : this.observationByHash.values()) { String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); - if( observationVariableName!=null && traitMap.containsKey(observationVariableName)){ + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); - observation.getBrAPIObject().setObservationVariableDbId( observationVariableDbId ); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); } } } @@ -752,94 +789,113 @@ private List getTraitList(Program program) { // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String programKey) { // FILTER LOGIC: Match on Env and Exp Unit ID - this.observationByHash.values().stream() - .filter(obs -> - obs.getBrAPIObject().getAdditionalInfo() != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null - && obs.getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.STUDY_NAME) - .getAsString() - .equals(obsUnit.getStudyName()) - && Utilities.removeProgramKeyAndUnknownAdditionalData( - obs.getBrAPIObject().getObservationUnitName(), programKey) - .equals( - Utilities.removeProgramKeyAndUnknownAdditionalData( - obsUnit.getObservationUnitName(), programKey) - ) - ) - .forEach(obs -> { - if(StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { - obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); - } - obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); - obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); - }); + this.observationByHash.values() + .stream() + .filter(obs -> obs.getBrAPIObject() + .getAdditionalInfo() != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) != null + && obs.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.STUDY_NAME) + .getAsString() + .equals(obsUnit.getStudyName()) + && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) + ) + .forEach(obs -> { + if (StringUtils.isBlank(obs.getBrAPIObject().getObservationUnitDbId())) { + obs.getBrAPIObject().setObservationUnitDbId(obsUnit.getObservationUnitDbId()); + } + obs.getBrAPIObject().setStudyDbId(obsUnit.getStudyDbId()); + obs.getBrAPIObject().setGermplasmDbId(obsUnit.getGermplasmDbId()); + }); } private void updateObsUnitDependencyValues(String programKey) { // update study DbIds - this.studyByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); // update germplasm DbIds - this.existingGermplasmByGID.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); } private void updateStudyDbId(BrAPIStudy study, String programKey) { - this.observationUnitByNameNoScope.values().stream() - .filter(obsUnit -> obsUnit.getBrAPIObject().getStudyName().equals( Utilities.removeProgramKeyAndUnknownAdditionalData( study.getStudyName(), programKey ) )) - .forEach(obsUnit -> { - obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId()); - obsUnit.getBrAPIObject().setTrialDbId(study.getTrialDbId()); - }); + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); } private void updateGermplasmDbId(BrAPIGermplasm germplasm) { - this.observationUnitByNameNoScope.values().stream() - .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && - obsUnit.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) - .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getGermplasmName() != null && + obsUnit.getBrAPIObject() + .getGermplasmName() + .equals(germplasm.getGermplasmName())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); } private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations - mappedBrAPIImport.values().stream() - .map(PendingImport::getLocation) - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateStudyLocationDbId); + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateStudyLocationDbId); // update trial DbIds in studies for all distinct trials - this.trialByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(trait -> this.updateTrialDbId(trait, programKey)); + this.trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trait -> this.updateTrialDbId(trait, programKey)); } private void updateStudyLocationDbId(BrAPILocation location) { - this.studyByNameNoScope.values().stream() - .filter(study -> location.getLocationName().equals( study.getBrAPIObject().getLocationName() )) - .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); + this.studyByNameNoScope.values() + .stream() + .filter(study -> location.getLocationName() + .equals(study.getBrAPIObject() + .getLocationName())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getLocationDbId())); } private void updateTrialDbId(BrAPITrial trial, String programKey) { - this.studyByNameNoScope.values().stream() - .filter(study -> study.getBrAPIObject().getTrialName().equals(Utilities.removeProgramKey(trial.getTrialName(), programKey ) ) ) - .forEach(study -> study.getBrAPIObject().setTrialDbId(trial.getTrialDbId())); + this.studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); } @Override @@ -853,9 +909,10 @@ private ArrayList getGermplasmByAccessionNumber( List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); ArrayList resultGermplasm = new ArrayList<>(); // Search for accession number matches - for (BrAPIGermplasm germplasm: germplasmList) { - for (String accessionNumber: germplasmAccessionNumbers) { - if (germplasm.getAccessionNumber().equals(accessionNumber)) { + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { resultGermplasm.add(germplasm); break; } @@ -864,12 +921,12 @@ private ArrayList getGermplasmByAccessionNumber( return resultGermplasm; } - private Map> initialize_existingGermplasmByGID(Program program, List experimentImportRows) { + private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); List uniqueGermplasmGIDs = experimentImportRows.stream() - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); List existingGermplasms; try { @@ -882,9 +939,9 @@ private Map> initialize_existingGerm } } - private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { + private Map> initializeStudyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); - if( this.trialByNameNoScope.size()!=1){ + if (this.trialByNameNoScope.size() != 1) { return studyByNameNoScope; } ExperimentObservation experimentObservation = experimentImportRows.get(0); @@ -899,7 +956,7 @@ private Map> initialize_studyByNameNoSco existingStudies.forEach(existingStudy -> { //Swap season DbId with year String String seasonId = existingStudy.getSeasons().get(0); - existingStudy.setSeasons( List.of( this.seasonDbIdToYear( seasonId, program.getId() ) ) ); + existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); studyByNameNoScope.put( @@ -913,13 +970,13 @@ private Map> initialize_studyByNameNoSco } } - private Map> initialize_uniqueLocationNames(Program program, List experimentImportRows) { + private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { Map> locationByName = new HashMap<>(); List uniqueLocationNames = experimentImportRows.stream() - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); List existingLocations; try { @@ -934,30 +991,29 @@ private Map> initialize_uniqueLocatio } } - private Map> initialize_trialByNameNoScope(Program program, List experimentImportRows) { + private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { Map> trialByNameNoScope = new HashMap<>(); String programKey = program.getKey(); List uniqueTrialNames = experimentImportRows.stream() - .map(experimentImport -> Utilities.appendProgramKey( experimentImport.getExpTitle(), programKey, null) ) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey, null)) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); List existingTrials; try { existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); existingTrials.forEach(existingTrial -> { existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); //get TrialId from existingTrial - List experimentRefs = existingTrial.getExternalReferences(); - Optional experimentIDRef = experimentRefs.stream() - .filter(this::isTrialRefSource) - .findFirst(); - if( experimentIDRef.isEmpty()){ - throw new InternalServerException("An Experiment ID was not found any of the external references"); + Optional experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource); + if (experimentIDRef.isEmpty()) { + throw new InternalServerException("An Experiment ID was not found in any of the external references"); } - UUID experimentId = UUID.fromString( experimentIDRef.get().getReferenceID() ); + UUID experimentId = UUID.fromString(experimentIDRef.get() + .getReferenceID()); trialByNameNoScope.put( existingTrial.getTrialName(), @@ -969,13 +1025,56 @@ private Map> initialize_trialByNameNoSco throw new InternalServerException(e.toString(), e); } } - private String simpleStudyName(String scopedName){ + + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> ret = new HashMap<>(); + + Map obsUnitById = new HashMap<>(); + + experimentImportRows.forEach(experimentObservation -> { + if (StringUtils.isNotBlank(experimentObservation.getObsUnitID())) { + obsUnitById.put(experimentObservation.getObsUnitID(), experimentObservation); + } + }); + + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(obsUnitById.keySet(), program); + + if (existingObsUnits.size() != obsUnitById.size()) { + //TODO figure out which of the OUs wasn't found + } else { + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + existingObsUnits.forEach(brAPIObservationUnit -> { + Optional idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource); + if (idRef.isPresent()) { + ExperimentObservation experimentObservation = obsUnitById.get(idRef.get() + .getReferenceID()); + experimentObservation.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + ret.put(createObservationUnitKey(experimentObservation.getEnv(), experimentObservation.getExpUnitId()), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.get() + .getReferenceID()))); + } else { + throw new InternalServerException("An ObservationUnit ID was not found in any of the external references"); + } + }); + } + + return ret; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private String simpleStudyName(String scopedName) { return scopedName.replaceFirst(" \\[.*\\]", ""); } private void validateTimeStampValue(String value, - String columnHeader, ValidationErrors validationErrors, int row){ - if(StringUtils.isBlank(value)) { + String columnHeader, ValidationErrors validationErrors, int row) { + if (StringUtils.isBlank(value)) { log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); return; } @@ -984,20 +1083,20 @@ private void validateTimeStampValue(String value, } } + private void validateObservationValue(Trait variable, String value, String columnHeader, ValidationErrors validationErrors, int row) { - if(StringUtils.isBlank(value)) { + if (StringUtils.isBlank(value)) { log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s\n\trow: %d", variable.getObservationVariableName(), row)); return; } - switch(variable.getScale().getDataType()) { + switch (variable.getScale().getDataType()) { case NUMERICAL: Optional number = validNumericValue(value); if (number.isEmpty()) { addRowError(columnHeader, "Non-numeric text detected detected", validationErrors, row); - } - else if (!validNumericRange(number.get(), variable.getScale())) { + } else if (!validNumericRange(number.get(), variable.getScale())) { addRowError(columnHeader, "Value outside of min/max range detected", validationErrors, row); } break; @@ -1021,6 +1120,7 @@ else if (!validNumericRange(number.get(), variable.getScale())) { } } + private Optional validNumericValue(String value) { BigDecimal number; try { @@ -1034,7 +1134,7 @@ private Optional validNumericValue(String value) { private boolean validNumericRange(BigDecimal value, Scale validValues) { // account for empty min or max in valid determination return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && - (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); } private boolean validDateValue(String value) { @@ -1058,7 +1158,9 @@ private boolean validDateTimeValue(String value) { } private boolean validCategory(List categories, String value) { - Set categoryValues = categories.stream().map(category -> category.getValue().toLowerCase()).collect(Collectors.toSet()); + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); return categoryValues.contains(value.toLowerCase()); } @@ -1068,30 +1170,27 @@ private boolean validCategory(List categories, * NOTE: This assumes that the only Season records of interest are ones * with a blank name or a name that is the same as the year. * - * @param year The year as a string + * @param year The year as a string * @param programId the program ID. * @return the DbId of the season-record associated with the year */ private String yearToSeasonDbId(String year, UUID programId) { String dbID = null; - if (this.yearToSeasonDbIdCache.containsKey(year) ){ // get it from cache if possible + if (this.yearToSeasonDbIdCache.containsKey(year)) { // get it from cache if possible dbID = this.yearToSeasonDbIdCache.get(year); - } - else{ - dbID = this.yearToSeasonDbIdFromDatabase(year,programId); + } else { + dbID = this.yearToSeasonDbIdFromDatabase(year, programId); this.yearToSeasonDbIdCache.put(year, dbID); } return dbID; } - private String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; - if (this.seasonDbIdToYearCache.containsKey(seasonDbId) ){ // get it from cache if possible + if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible year = this.seasonDbIdToYearCache.get(seasonDbId); - } - else{ - year = this.seasonDbIdToYearFromDatabase(seasonDbId,programId); + } else { + year = this.seasonDbIdToYearFromDatabase(seasonDbId, programId); this.seasonDbIdToYearCache.put(seasonDbId, year); } return year; @@ -1102,17 +1201,17 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { List seasons; try { seasons = this.brAPISeasonDAO.getSeasonByYear(year, programId); - for( BrAPISeason season : seasons){ - if(null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)){ + for (BrAPISeason season : seasons) { + if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { targetSeason = season; break; } } - if (targetSeason == null){ + if (targetSeason == null) { BrAPISeason newSeason = new BrAPISeason(); - newSeason.setYear( Integer.parseInt(year) ); - newSeason.setSeasonName( year ); - targetSeason = this.brAPISeasonDAO.addOneSeason( newSeason, programId ); + newSeason.setYear(Integer.parseInt(year)); + newSeason.setSeasonName(year); + targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); } } catch (ApiException e) { @@ -1120,7 +1219,7 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { log.error(e.getResponseBody(), e); } - String seasonDbId = (targetSeason==null) ? null : targetSeason.getSeasonDbId(); + String seasonDbId = (targetSeason == null) ? null : targetSeason.getSeasonDbId(); return seasonDbId; } @@ -1128,16 +1227,12 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { BrAPISeason targetSeason = null; BrAPISeason season = null; try { - season = this.brAPISeasonDAO.getSeasonById (seasonDbId, programId); + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); } catch (ApiException e) { log.error(e.getResponseBody(), e); } Integer yearInt = (season == null) ? null : season.getYear(); - String yearStr = (yearInt==null) ? "" : yearInt.toString(); + String yearStr = (yearInt == null) ? "" : yearInt.toString(); return yearStr; } - - private boolean isTrialRefSource(BrAPIExternalReference brAPIExternalReference) { - return brAPIExternalReference.getReferenceSource().equals( Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS)); - } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 12084c9de..139fbf37e 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -120,4 +121,11 @@ public static String generateApiExceptionLogMessage(ApiException e) { public static String generateReferenceSource(String referenceSourceBase, ExternalReferenceSource referenceSource) { return String.format("%s/%s",referenceSourceBase, referenceSource.getName()); } + + public static Optional getExternalReference(List externalReferences, String source) { + if(externalReferences == null) { + return Optional.empty(); + } + return externalReferences.stream().filter(externalReference -> externalReference.getReferenceSource().equals(source)).findFirst(); + } } From ca0ed4961e5f33d02f1ede857cc456d3558ab738 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 25 Jan 2023 00:35:38 -0500 Subject: [PATCH 02/15] [BI-1195] Completed initial implementation of supporting appending of observations to existing OUs Also refactored the ExperimentProcessor to improve readability/maintainability --- .../brapi/v2/GermplasmController.java | 18 +- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 13 +- .../v2/services/BrAPIGermplasmService.java | 8 +- .../importer/daos/BrAPILocationDAO.java | 17 +- .../importer/daos/BrAPIObservationDAO.java | 3 +- .../brapps/importer/daos/BrAPISeasonDAO.java | 23 +- .../brapps/importer/daos/BrAPIStudyDAO.java | 35 +- .../brapps/importer/daos/BrAPITrialDAO.java | 32 +- .../ExperimentObservation.java | 143 +++-- .../processors/ExperimentProcessor.java | 606 +++++++++++------- .../processors/LocationProcessor.java | 2 +- .../processors/ObservationProcessor.java | 2 +- .../services/processors/StudyProcessor.java | 4 +- .../services/processors/TrialProcessor.java | 4 +- 14 files changed, 533 insertions(+), 377 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index d185ee3f3..dae16e60c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -30,6 +30,7 @@ import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.model.request.query.GermplasmQuery; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.model.exports.FileType; @@ -44,10 +45,7 @@ import javax.inject.Inject; import javax.validation.Valid; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; +import java.util.*; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -205,7 +203,12 @@ public HttpResponse getGermplasmPedigreeInfo( metadata.setPagination(pagination); response = new BrAPIGermplasmPedigreeResponse(); } else { - BrAPIGermplasm germplasm = germplasmService.getGermplasmByDBID(programId, germplasmId); + Optional germplasmOptional = germplasmService.getGermplasmByDBID(programId, germplasmId); + if(germplasmOptional.isEmpty()) { + throw new DoesNotExistException("DBID for this germplasm does not exist"); + } + + BrAPIGermplasm germplasm = germplasmOptional.get(); //Forward the pedigree call to the backing BrAPI system of the program passing the germplasmDbId that came in the request GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); ApiResponse pedigreeResponse = api.germplasmGermplasmDbIdPedigreeGet(germplasmId, notation, includeSiblings); @@ -238,12 +241,15 @@ public HttpResponse getGermplasmPedigreeInfo( response.setResult(returnNode); response.setMetadata(metadata); return HttpResponse.ok(response); - } catch (InternalServerException | ApiException e) { + } catch (InternalServerException e) { log.error(e.getMessage(), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); return HttpResponse.status(HttpStatus.NOT_FOUND, "Pedigree node not found"); + } catch (ApiException e) { + log.info(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 280e31bf3..f18d730d2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -273,15 +273,22 @@ public BrAPIGermplasm getGermplasmByUUID(String germplasmId, UUID programId) thr return germplasm; } - public BrAPIGermplasm getGermplasmByDBID(String germplasmDbId, UUID programId) throws ApiException, DoesNotExistException { + public Optional getGermplasmByDBID(String germplasmDbId, UUID programId) throws ApiException { Map cache = programGermplasmCache.get(programId); //key is UUID, want to filter by DBID BrAPIGermplasm germplasm = null; if (cache != null) { germplasm = cache.values().stream().filter(x -> x.getGermplasmDbId().equals(germplasmDbId)).collect(Collectors.toList()).get(0); } - if (germplasm == null) { - throw new DoesNotExistException("DBID for this germplasm does not exist"); + return Optional.ofNullable(germplasm); + } + + public List getGermplasmsByDBID(Set germplasmDbIds, UUID programId) throws ApiException { + Map cache = programGermplasmCache.get(programId); + //key is UUID, want to filter by DBID + List germplasm = new ArrayList<>(); + if (cache != null) { + germplasm = cache.values().stream().filter(x -> germplasmDbIds.contains(x.getGermplasmDbId())).collect(Collectors.toList()); } return germplasm; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 03451b43b..d09848c91 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -73,12 +73,8 @@ public BrAPIGermplasm getGermplasmByUUID(UUID programId, String germplasmId) thr } } - public BrAPIGermplasm getGermplasmByDBID(UUID programId, String germplasmId) throws DoesNotExistException { - try { - return germplasmDAO.getGermplasmByDBID(germplasmId, programId); - } catch (ApiException e) { - throw new InternalServerException(e.getMessage(), e); - } + public Optional getGermplasmByDBID(UUID programId, String germplasmId) throws ApiException { + return germplasmDAO.getGermplasmByDBID(germplasmId, programId); } public List getGermplasmListsByProgramId(UUID programId, HttpRequest request) throws DoesNotExistException, ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java index 65ddeda54..68febcd3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java @@ -28,9 +28,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; @Singleton public class BrAPILocationDAO { @@ -61,9 +59,20 @@ public List getLocationsByName(List locationNames, UUID p ); } - public List createBrAPILocation(List brAPILocationList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPILocations(List brAPILocationList, UUID programId, ImportUpload upload) throws ApiException { LocationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); return brAPIDAOUtil.post(brAPILocationList, upload, api::locationsPost, importDAO::update); } + public List getLocationsByDbId(Collection locationDbIds, UUID programId) throws ApiException { + BrAPILocationSearchRequest locationSearchRequest = new BrAPILocationSearchRequest(); + locationSearchRequest.setLocationDbIds(new ArrayList<>(locationDbIds)); + //TODO: Locations don't connect to programs. How to get locations for the program? + LocationsApi api = new LocationsApi(programDAO.getCoreClient(programId)); + return brAPIDAOUtil.search( + api::searchLocationsPost, + api::searchLocationsSearchResultsDbIdGet, + locationSearchRequest + ); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 1115b625c..d2006644d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -25,7 +25,6 @@ import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.daos.ObservationDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -77,7 +76,7 @@ public List getObservationsByStudyName(List studyNames return api.searchObservationsSearchResultsDbIdGet(APPLICATION_JSON, searchResultsDbId, page, pageSize); } - public List createBrAPIObservation(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPIObservations(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); return brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java index df7965c93..0ca2600b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java @@ -4,28 +4,16 @@ import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.SeasonQueryParams; -import org.brapi.client.v2.modules.core.ListsApi; import org.brapi.client.v2.modules.core.SeasonsApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.BrAPIResponse; -import org.brapi.v2.model.BrAPIResponseResult; -import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPISeason; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.request.BrAPIListSearchRequest; -import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.core.response.BrAPISeasonListResponse; import org.brapi.v2.model.core.response.BrAPISeasonListResponseResult; import org.brapi.v2.model.core.response.BrAPISeasonSingleResponse; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; import javax.inject.Inject; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -34,30 +22,25 @@ public class BrAPISeasonDAO { private ProgramDAO programDAO; - private ImportDAO importDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; @Inject - public BrAPISeasonDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPISeasonDAO(ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider) { this.programDAO = programDAO; - this.importDAO = importDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getSeasonByYear(String year, UUID programId) throws ApiException { + public List getSeasonsByYear(String year, UUID programId) throws ApiException { SeasonsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), SeasonsApi.class); SeasonQueryParams queryParams = SeasonQueryParams.builder() .year( year ) .pageSize( 10000 ) .build(); - List seasons = new ArrayList<>(); ApiResponse apiResponse = api.seasonsGet( queryParams ); BrAPISeasonListResponse seasonListResponse = apiResponse.getBody(); BrAPISeasonListResponseResult result = seasonListResponse.getResult(); - seasons = result.getData(); - - return seasons; + return result.getData(); } public BrAPISeason getSeasonById(String id, UUID programId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index f0277fc08..04c30a52c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -31,8 +31,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.List; -import java.util.UUID; +import java.util.*; @Singleton public class BrAPIStudyDAO { @@ -52,7 +51,15 @@ public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getStudyByName(List studyNames, Program program) throws ApiException { + public Optional getStudyByName(String studyName, Program program) throws ApiException { + List studies = getStudiesByName(List.of(studyName), program); + if(studies.size() == 1) { + return Optional.of(studies.get(0)); + } else { + return Optional.empty(); + } + } + public List getStudiesByName(List studyNames, Program program) throws ApiException { BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.studyNames(studyNames); @@ -77,9 +84,29 @@ public List getStudiesByExperimentID(UUID experimentID, Program prog ); } - public List createBrAPIStudy(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPIStudies(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); return brAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); } + public List getStudiesByStudyDbId(Collection studyDbIds, Program program) throws ApiException { + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); + studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + studySearch.studyDbIds(new ArrayList<>(studyDbIds)); + StudiesApi api = new StudiesApi(programDAO.getCoreClient(program.getId())); + return brAPIDAOUtil.search( + api::searchStudiesPost, + api::searchStudiesSearchResultsDbIdGet, + studySearch + ); + } + + public Optional getStudyByDbId(String studyDbId, Program program) throws ApiException { + List studies = getStudiesByStudyDbId(List.of(studyDbId), program); + if(studies.size() == 1) { + return Optional.of(studies.get(0)); + } else { + return Optional.empty(); + } + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index fe7d16ad6..8c0deb30d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -18,11 +18,8 @@ import io.micronaut.context.annotation.Property; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.modules.core.StudiesApi; import org.brapi.client.v2.modules.core.TrialsApi; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; @@ -36,10 +33,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; @Singleton public class BrAPITrialDAO { @@ -62,7 +56,7 @@ public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br this.brAPIEndpointProvider = brAPIEndpointProvider; } - public List getTrialByName(List trialNames, Program program) throws ApiException { + public List getTrialsByName(List trialNames, Program program) throws ApiException { BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); trialSearch.trialNames(trialNames); @@ -74,7 +68,7 @@ public List getTrialByName(List trialNames, Program program) ); } - public List createBrAPITrial(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); return brAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); } @@ -124,6 +118,26 @@ private List processExperimentsForDisplay(List trials, S return displayExperiments; } + public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { + List trials = getTrialsByDbIds(List.of(trialDbId), program); + if(trials.size() == 1) { + return Optional.of(trials.get(0)); + } else { + return Optional.empty(); + } + } + + public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + trialSearch.trialDbIds(new ArrayList<>(trialDbIds)); + TrialsApi api = new TrialsApi(programDAO.getCoreClient(program.getId())); + return brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 1c50402a2..722cb25fb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -42,76 +42,76 @@ @Getter @Setter @NoArgsConstructor -@ImportConfigMetadata(id="ExperimentImport", name="Experiment Import", +@ImportConfigMetadata(id = "ExperimentImport", name = "Experiment Import", description = "This import is used to create Observation Unit and Experiment data") public class ExperimentObservation implements BrAPIImport { - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="germplasmName", name="Germplasm Name", description = "Name of germplasm") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "germplasmName", name = "Germplasm Name", description = "Name of germplasm") private String germplasmName; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="gid", name="Germplasm GID", description = "Unique germplasm identifier") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "gid", name = "Germplasm GID", description = "Unique germplasm identifier") private String gid; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="test_or_check", name="Test or Check", description = "T test (T) and check (C) germplasm") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "test_or_check", name = "Test or Check", description = "T test (T) and check (C) germplasm") private String testOrCheck; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="exp_title", name="Experiment Title", description = "Title of experiment") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "exp_title", name = "Experiment Title", description = "Title of experiment") private String expTitle; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expDescription", name="Experiment Description", description = "Description of experiment") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expDescription", name = "Experiment Description", description = "Description of experiment") private String expDescription; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expUnit", name="Experiment Unit", description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expUnit", name = "Experiment Unit", description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") private String expUnit; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expType", name="Experiment Type", description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expType", name = "Experiment Type", description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") private String expType; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="env", name="Environment", description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "env", name = "Environment", description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") private String env; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="envLocation", name="Environment Location", description = "Location of the environment") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "envLocation", name = "Environment Location", description = "Location of the environment") private String envLocation; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id="envYear", name="Environment Year", description = "Year corresponding to the environment") + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id = "envYear", name = "Environment Year", description = "Year corresponding to the environment") private String envYear; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="expUnitId", name="Experiment Unit ID", description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "expUnitId", name = "Experiment Unit ID", description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") private String expUnitId; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id="expReplicateNo", name="Experiment Replicate Number", description = "Sequential number of experimental replications") + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id = "expReplicateNo", name = "Experiment Replicate Number", description = "Sequential number of experimental replications") private String expReplicateNo; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id="expBlockNo", name="Experiment Block Number", description = "Sequential number of blocks in an experimental design") + @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id = "expBlockNo", name = "Experiment Block Number", description = "Sequential number of blocks in an experimental design") private String expBlockNo; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="row", name="Row", description = "Horizontal (y-axis) position in 2D Cartesian space.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "row", name = "Row", description = "Horizontal (y-axis) position in 2D Cartesian space.") private String row; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="column", name="Column", description = "Vertical (x-axis) position in 2D Cartesian space.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "column", name = "Column", description = "Vertical (x-axis) position in 2D Cartesian space.") private String column; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="treatmentFactors", name="Treatment Factors", description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "treatmentFactors", name = "Treatment Factors", description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") private String treatmentFactors; - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id="obsUnitID", name="Observation Unit ID", description = "A database generated unique identifier for experimental observation units") + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "obsUnitID", name = "Observation Unit ID", description = "A database generated unique identifier for experimental observation units") private String obsUnitID; public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit, String referenceSource, UUID id, String expSeqValue) { @@ -119,9 +119,8 @@ public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit BrAPITrial trial = new BrAPITrial(); if (commit) { setBrAPITrialCommitFields(program, trial, referenceSource, id); - } - else{ - trial.setTrialName( getExpTitle() ); + } else { + trial.setTrialName(getExpTitle()); } trial.setTrialDescription(getExpDescription()); trial.setActive(true); @@ -129,18 +128,20 @@ public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit trial.setProgramName(brapiProgram.getProgramName()); Map createdBy = new HashMap<>(); - createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, user.getId().toString()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, + user.getId() + .toString()); createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy); - trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); - trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); - trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); + trial.putAdditionalInfoItem(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); return trial; } private void setBrAPITrialCommitFields(Program program, BrAPITrial trial, String referenceSource, UUID id) { - trial.setTrialName( Utilities.appendProgramKey(getExpTitle(), program.getKey() )); + trial.setTrialName(Utilities.appendProgramKey(getExpTitle(), program.getKey())); // Set external reference trial.setExternalReferences(getTrialExternalReferences(program, referenceSource, id)); @@ -166,13 +167,12 @@ public BrAPIStudy constructBrAPIStudy( UUID id, Supplier envNextVal) { BrAPIStudy study = new BrAPIStudy(); - if ( commit ){ + if (commit) { study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), expSequenceValue)); // Set external reference study.setExternalReferences(getStudyExternalReferences(program, referenceSource, trialId, id)); - } - else { + } else { study.setStudyName(getEnv()); } study.setActive(true); @@ -181,20 +181,20 @@ public BrAPIStudy constructBrAPIStudy( study.setTrialName(getExpTitle()); List seasonList = new ArrayList<>(); - seasonList.add( getEnvYear() ); - study.setSeasons( seasonList ); + seasonList.add(getEnvYear()); + study.setSeasons(seasonList); String designType = "Analysis"; // to support the BRApi server, the design type must be one of the following: - // 'CRD','Alpha','MAD','Lattice','Augmented','RCBD','p-rep','splitplot','greenhouse','Westcott', or 'Analysis' - // For now it will be hardcoded to 'Analysis' + // 'CRD','Alpha','MAD','Lattice','Augmented','RCBD','p-rep','splitplot','greenhouse','Westcott', or 'Analysis' + // For now it will be hardcoded to 'Analysis' BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); design.setPUI(designType); design.setDescription(designType); study.setExperimentalDesign(design); String envSequenceValue = null; - if( commit ){ + if (commit) { envSequenceValue = envNextVal.get().toString(); - study.putAdditionalInfoItem( BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); + study.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); } return study; } @@ -211,18 +211,17 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( ) { BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); - if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); + if (commit) { + observationUnit.setObservationUnitName(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); // Set external reference observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); - } - else { + } else { observationUnit.setObservationUnitName(getExpUnitId()); } observationUnit.setStudyName(getEnv()); - if(germplasmName==null){ + if (germplasmName == null) { germplasmName = getGermplasmName(); } observationUnit.setGermplasmName(germplasmName); @@ -230,13 +229,13 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" - level.setLevelCode( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal) ); + level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); // Exp Unit List levelRelationships = new ArrayList<>(); - if( getExpReplicateNo() !=null ) { + if (getExpReplicateNo() != null) { BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); repLvl.setLevelName(BrAPIConstants.REPLICATE.getValue()); repLvl.setLevelCode(getExpReplicateNo()); @@ -244,16 +243,16 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } // Block number - if( getExpBlockNo() != null ) { + if (getExpBlockNo() != null) { BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); - repLvl.setLevelName( BrAPIConstants.BLOCK.getValue() ); + repLvl.setLevelName(BrAPIConstants.BLOCK.getValue()); repLvl.setLevelCode(getExpBlockNo()); levelRelationships.add(repLvl); } position.setObservationLevelRelationships(levelRelationships); // Test or Check - if("C".equals(getTestOrCheck())){ + if ("C".equals(getTestOrCheck())) { position.setEntryType(BrAPIEntryTypeEnum.CHECK); } else { position.setEntryType(BrAPIEntryTypeEnum.TEST); @@ -289,10 +288,10 @@ public BrAPIObservation constructBrAPIObservation( String variableName, String seasonDbId, BrAPIObservationUnit obsUnit - ) { + ) { BrAPIObservation observation = new BrAPIObservation(); observation.setGermplasmName(getGermplasmName()); - if(getEnv() != null) { + if (getEnv() != null) { observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); } observation.setObservationVariableName(variableName); @@ -313,9 +312,15 @@ private List getBrAPIExternalReferences( List refs = new ArrayList<>(); addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); - if( trialId != null ) { addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } - if( studyId != null ) { addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } - if( obsUnitId != null ) { addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); } + if (trialId != null) { + addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); + } + if (studyId != null) { + addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); + } + if (obsUnitId != null) { + addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); + } return refs; } @@ -324,10 +329,12 @@ private List getTrialExternalReferences( Program program, String referenceSourceBaseName, UUID trialId) { return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null); } + private List getStudyExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId) { return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); } + private List getObsUnitExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); @@ -337,7 +344,7 @@ private List getObsUnitExternalReferences( private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { BrAPIExternalReference reference; reference = new BrAPIExternalReference(); - reference.setReferenceSource( String.format("%s/%s", referenceBaseNameSource, refSourceName.getName()) ); + reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); reference.setReferenceID(uuid.toString()); refs.add(reference); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 414a503e1..21215ec71 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -88,16 +88,16 @@ public class ExperimentProcessor implements Processor { @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; - private DSLContext dsl; - private BrAPITrialDAO brapiTrialDAO; - private BrAPILocationDAO brAPILocationDAO; - private BrAPIStudyDAO brAPIStudyDAO; - private BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private BrAPIObservationDAO brAPIObservationDAO; - private BrAPISeasonDAO brAPISeasonDAO; - private BrAPIGermplasmDAO brAPIGermplasmDAO; - private OntologyService ontologyService; - private FileMappingUtil fileMappingUtil; + private final DSLContext dsl; + private final BrAPITrialDAO brapiTrialDAO; + private final BrAPILocationDAO brAPILocationDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPIObservationDAO brAPIObservationDAO; + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final OntologyService ontologyService; + private final FileMappingUtil fileMappingUtil; // used to make the yearsToSeasonDbId() function more efficient private final Map yearToSeasonDbIdCache = new HashMap<>(); @@ -119,7 +119,7 @@ public class ExperimentProcessor implements Processor { private Map> existingGermplasmByGID = null; // Associates timestamp columns to associated phenotype column name for ease of storage - private Map timeStampColByPheno = new HashMap<>(); + private Map> timeStampColByPheno = new HashMap<>(); @Inject public ExperimentProcessor(DSLContext dsl, @@ -144,6 +144,11 @@ public ExperimentProcessor(DSLContext dsl, this.fileMappingUtil = fileMappingUtil; } + @Override + public String getName() { + return NAME; + } + /** * Initialize the Map objects with existing BrAPI Data. * @@ -157,11 +162,10 @@ public void getExistingBrapiData(List importRows, Program program) .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); + this.observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); this.trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); this.studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - this.observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - // key and removing key + this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); this.existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); } @@ -190,7 +194,7 @@ public Map process( List> dynamicCols = fileMappingUtil.getDynamicColumns(data, EXPERIMENT_TEMPLATE_NAME); List> phenotypeCols = new ArrayList<>(); List> timestampCols = new ArrayList<>(); - for (Column dynamicCol : dynamicCols) { + for (Column dynamicCol : dynamicCols) { //Distinguish between phenotype and timestamp columns if (dynamicCol.name().startsWith("TS:")) { timestampCols.add(dynamicCol); @@ -202,15 +206,95 @@ public Map process( validateObservationAndTimestampColumns(program.getId(), phenotypeCols, timestampCols, validationErrors); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - for (Column tsColumn : timestampCols) { - timeStampColByPheno.put(tsColumn.name() - .replaceFirst("^TS:\\s*", ""), tsColumn); + for (Column tsColumn : timestampCols) { + timeStampColByPheno.put(tsColumn.name().replaceFirst("^TS:\\s*", ""), tsColumn); } // add "New" pending data to the BrapiData objects - getNewBrapiData(importRows, phenotypeCols, program, user, commit); + initNewBrapiData(importRows, phenotypeCols, program, user, commit); + + prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport, validationErrors); + + validateFields(importRows, validationErrors); + + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + // Construct our response object + return getStatisticsMap(importRows); + } + + @Override + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } + + @Override + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + + List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List newLocations = ProcessorData.getNewObjects(this.locationByName); + List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + try { + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + this.trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); + } + + List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocations(newLocations, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPILocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getLocationName(); + this.locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); + } + + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + this.studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } - // For each import row + updateObsUnitDependencyValues(program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + this.observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + updateObservationDependencyValues(program); + brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + } catch (ApiException e) { + log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error saving experiment import", e); + } + } + + private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport, ValidationErrors validationErrors) throws MissingRequiredInfoException { for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); @@ -231,28 +315,18 @@ public Map process( PendingImportObject germplasmPIO = getGidPOI(importRow); mappedImportRow.setGermplasm(germplasmPIO); - if (!StringUtils.isBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. validateGermplasm(importRow, validationErrors, i, germplasmPIO); } //Check if existing environment. If so, ObsUnitId must be assigned - if ((this.studyByNameNoScope.get(importRow.getEnv()) - .getState() == ImportObjectState.EXISTING) && (StringUtils.isBlank(importRow.getObsUnitID()))) { + if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) + && (StringUtils.isBlank(importRow.getObsUnitID()))) { throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); } mappedBrAPIImport.put(i, mappedImportRow); } - // End-of-loop - - validationErrors = validateFields(importRows, validationErrors); - - if (validationErrors.hasErrors()) { - throw new ValidatorException(validationErrors); - } - - // Construct our response object - return getStatisticsMap(importRows); } private void validateObservationAndTimestampColumns(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { @@ -329,7 +403,7 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void getNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, boolean commit) { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, boolean commit) { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -346,11 +420,9 @@ private void getNewBrapiData(List importRows, List> pheno Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); for (int i = 0; i < importRows.size(); i++) { - BrAPIImport row = importRows.get(i); - ExperimentObservation importRow = (ExperimentObservation) row; + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - PendingImportObject trialPIO = createTrialPIO(program, user, commit, importRow, expNextVal); - this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); + PendingImportObject trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); String expSeqValue = null; if (commit) { @@ -360,11 +432,9 @@ private void getNewBrapiData(List importRows, List> pheno .getAsString(); } - PendingImportObject locationPIO = createLocationPIO(importRow); - this.locationByName.put(importRow.getEnvLocation(), locationPIO); + fetchOrCreateLocationPIO(importRow); - PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow, envNextVal); - this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); + PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); String envSeqValue = null; if (commit) { @@ -374,9 +444,7 @@ private void getNewBrapiData(List importRows, List> pheno .getAsString(); } - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, envSeqValue, importRow); - String key = createObservationUnitKey(importRow); - this.observationUnitByNameNoScope.put(key, obsUnitPIO); + PendingImportObject obsUnitPIO = fetchOrCreateObsUnitPIO(program, commit, envSeqValue, importRow); for (Column column : phenotypeCols) { //If associated timestamp column, add @@ -391,20 +459,17 @@ private void getNewBrapiData(List importRows, List> pheno } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - PendingImportObject obsPIO = createObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO); - this.observationByHash.put(getImportObservationHash(importRow, getVariableNameFromColumn(column)), obsPIO); + fetchOrCreateObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO); } } } private String createObservationUnitKey(ExperimentObservation importRow) { - String key = createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); - return key; + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); } private String createObservationUnitKey(String studyName, String obsUnitName) { - String key = studyName + obsUnitName; - return key; + return studyName + obsUnitName; } private String getImportObservationHash(ExperimentObservation importRow, String variableName) { @@ -566,7 +631,7 @@ private Map getStatisticsMap(List private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { // error if GID is not blank but GID does not already exist - if (!StringUtils.isBlank(importRow.getGid()) && germplasmPIO == null) { + if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { addRowError( "GID", "A non-existing GID", @@ -583,10 +648,11 @@ private PendingImportObject getGidPOI(ExperimentObservation impo } } - private PendingImportObject createObsUnitPIO(Program program, boolean commit, String seqValue, ExperimentObservation importRow) { - PendingImportObject pio = null; - if (this.observationUnitByNameNoScope.containsKey(createObservationUnitKey(importRow))) { - pio = observationUnitByNameNoScope.get(createObservationUnitKey(importRow)); + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (this.observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { @@ -599,23 +665,25 @@ private PendingImportObject createObsUnitPIO(Program progr PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, seqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + this.observationUnitByNameNoScope.put(key, pio); } return pio; } - private PendingImportObject createObservationPIO(ExperimentObservation importRow, - String variableName, - String value, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO) { - PendingImportObject pio = null; - if (this.observationByHash.containsKey(getImportObservationHash(importRow, variableName))) { - pio = observationByHash.get(getImportObservationHash(importRow, variableName)); + private PendingImportObject fetchOrCreateObservationPIO(ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO) { + PendingImportObject pio; + String key = getImportObservationHash(importRow, variableName); + if (this.observationByHash.containsKey(key)) { + pio = observationByHash.get(key); } else { BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject()); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. @@ -625,12 +693,13 @@ private PendingImportObject createObservationPIO(ExperimentObs } pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + this.observationByHash.put(key, pio); } return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { - PendingImportObject pio = null; + private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { + PendingImportObject pio; if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); } else { @@ -642,116 +711,44 @@ private PendingImportObject createStudyPIO(Program program, boolean if (commit) { String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Arrays.asList(seasonID)); + newStudy.setSeasons(Collections.singletonList(seasonID)); } pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); } return pio; } - private PendingImportObject createLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio = null; + private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; if (locationByName.containsKey((importRow.getEnvLocation()))) { pio = locationByName.get(importRow.getEnvLocation()); } else { BrAPILocation newLocation = importRow.constructBrAPILocation(); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + this.locationByName.put(importRow.getEnvLocation(), pio); } return pio; } - private PendingImportObject createTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { - PendingImportObject pio = null; + private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { + PendingImportObject pio; if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { pio = trialByNameNoScope.get(importRow.getExpTitle()); } else { UUID id = UUID.randomUUID(); String expSeqValue = null; if (commit) { - expSeqValue = expNextVal.get() - .toString(); + expSeqValue = expNextVal.get().toString(); } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + this.trialByNameNoScope.put(importRow.getExpTitle(), pio); } return pio; } - @Override - public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { - // TODO - } - - @Override - public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { - - List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); - List newLocations = ProcessorData.getNewObjects(this.locationByName); - List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); - List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - // filter out observations with no 'value' so they will not be saved - List newObservations = ProcessorData.getNewObjects(this.observationByHash) - .stream() - .filter(obs -> !obs.getValue().isBlank()) - .collect(Collectors.toList()); - try { - List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); - // set the DbId to the for each newly created trial - for (BrAPITrial createdTrial : createdTrials) { - String createdTrialName = createdTrial.getTrialName(); - String createdTrialName_no_key = Utilities.removeProgramKey(createdTrialName, program.getKey()); - PendingImportObject pi = this.trialByNameNoScope.get(createdTrialName_no_key); - BrAPITrial listedTrial = pi.getBrAPIObject(); - String dbid = createdTrial.getTrialDbId(); - listedTrial.setTrialDbId(dbid); - } - - List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload)); - // set the DbId to the for each newly created trial - for (BrAPILocation createdLocation : createdLocations) { - String createdLocationName = createdLocation.getLocationName(); - PendingImportObject pi = this.locationByName.get(createdLocationName); - BrAPILocation listedLocation = pi.getBrAPIObject(); - String dbid = createdLocation.getLocationDbId(); - listedLocation.setLocationDbId(dbid); - } - - updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); - List createdStudies = new ArrayList<>(); - createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(newStudies, program.getId(), upload)); - - // set the DbId to the for each newly created study - for (BrAPIStudy createdStudy : createdStudies) { - String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); - PendingImportObject pi = this.studyByNameNoScope.get(createdStudy_name_no_key); - BrAPIStudy brAPIStudy = pi.getBrAPIObject(); - brAPIStudy.setStudyDbId(createdStudy.getStudyDbId()); - } - - updateObsUnitDependencyValues(program.getKey()); - List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); - - // set the DbId to the for each newly created Observation Unit - for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { - // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope - String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); - String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); - String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); - PendingImportObject pi = this.observationUnitByNameNoScope.get(createdObsUnit_key); - // update the retrieved BrAPI object - BrAPIObservationUnit brAPIObservationUnit = pi.getBrAPIObject(); - brAPIObservationUnit.setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); - } - - updateObservationDependencyValues(program); - brAPIObservationDAO.createBrAPIObservation(newObservations, program.getId(), upload); - } catch (ApiException e) { - log.error(e.getResponseBody()); - throw new InternalServerException(e.toString(), e); - } - } - private void updateObservationDependencyValues(Program program) { String programKey = program.getKey(); @@ -764,7 +761,7 @@ private void updateObservationDependencyValues(Program program) { // Update ObservationVariable DbIds List traits = getTraitList(program); - Map traitMap = traits.stream().collect(Collectors.toMap(trait -> trait.getObservationVariableName(), trait -> trait)); + Map traitMap = traits.stream().collect(Collectors.toMap(TraitEntity::getObservationVariableName, trait -> trait)); for (PendingImportObject observation : this.observationByHash.values()) { String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); @@ -898,11 +895,6 @@ private void updateTrialDbId(BrAPITrial trial, String programKey) { .setTrialDbId(trial.getTrialDbId())); } - @Override - public String getName() { - return NAME; - } - private ArrayList getGermplasmByAccessionNumber( List germplasmAccessionNumbers, UUID programId) throws ApiException { @@ -923,138 +915,261 @@ private ArrayList getGermplasmByAccessionNumber( private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); - List uniqueGermplasmGIDs = experimentImportRows.stream() - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); List existingGermplasms; - try { - existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); - existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); - return existingGermplasmByGID; - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms = brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId()); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } else { + List uniqueGermplasmGIDs = experimentImportRows.stream() + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } } + + existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); + return existingGermplasmByGID; } private Map> initializeStudyByNameNoScope(Program program, List experimentImportRows) { - Map> studyByNameNoScope = new HashMap<>(); + Map> studyByName = new HashMap<>(); if (this.trialByNameNoScope.size() != 1) { - return studyByNameNoScope; + return studyByName; } - ExperimentObservation experimentObservation = experimentImportRows.get(0); - PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); + List existingStudies; + if(observationUnitByNameNoScope.size() > 0) { + List studyDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getStudyDbId()).collect(Collectors.toList()); + try { + existingStudies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } - UUID experimentId = trial.getId(); + if(existingStudies.size() < studyDbIds.size()) { + throw new IllegalStateException("Did not find all of the studies for the requested list of studyDbIds"); + } + } else { + //TODO rethink this somewhat...should there be support for both fetching studies from existing OUs and new OUs? + ExperimentObservation experimentObservation = experimentImportRows.get(0); - List existingStudies; - try { - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - existingStudies.forEach(existingStudy -> { - //Swap season DbId with year String - String seasonId = existingStudy.getSeasons().get(0); - existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); - - existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); - studyByNameNoScope.put( - existingStudy.getStudyName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); - }); - return studyByNameNoScope; - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); + PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); + + UUID experimentId = trial.getId(); + + + try { + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } } + + existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); + return studyByName; + } + + private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { + //Swap season DbId with year String + String seasonId = existingStudy.getSeasons().get(0); + existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); + + existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); + studyByName.put( + existingStudy.getStudyName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); } private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { Map> locationByName = new HashMap<>(); - List uniqueLocationNames = experimentImportRows.stream() - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); List existingLocations; - try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); - existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); - }); - return locationByName; - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values().stream().map(study -> study.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + try { + existingLocations = brAPILocationDAO.getLocationsByDbId(locationDbIds, program.getId()); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } else { + List uniqueLocationNames = experimentImportRows.stream() + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + + try { + existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } } + + existingLocations.forEach(existingLocation -> { + locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + }); + return locationByName; } private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { - Map> trialByNameNoScope = new HashMap<>(); + Map> trialByName = new HashMap<>(); String programKey = program.getKey(); + + initializeTrialsForExistingObservationUnits(program, experimentImportRows, trialByName); + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isNotBlank(row.getObsUnitID())) .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey, null)) .distinct() .filter(Objects::nonNull) .collect(Collectors.toList()); - List existingTrials; - try { - existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - existingTrials.forEach(existingTrial -> { - existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); + brapiTrialDAO.getTrialsByName(uniqueTrialNames, program) + .forEach(existingTrial -> processAndCacheTrial(existingTrial, program, trialRefSource, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } - //get TrialId from existingTrial - Optional experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource); - if (experimentIDRef.isEmpty()) { - throw new InternalServerException("An Experiment ID was not found in any of the external references"); + return trialByName; + } + + private void initializeTrialsForExistingObservationUnits(Program program, List experimentImportRows, Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isNotBlank(existingOu.getTrialDbId()) || StringUtils.isNotBlank(existingOu.getStudyDbId())) { + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + } else { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() == trialDbIds.size()) { + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); + trials.forEach(trial -> processAndCacheTrial(trial, program, trialRefSource, trialByName)); + } else { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(",", missingIds)); } - UUID experimentId = UUID.fromString(experimentIDRef.get() - .getReferenceID()); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } - trialByNameNoScope.put( - existingTrial.getTrialName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if(studies.size() == studyDbIds.size()) { + studies.forEach(study -> { + if (StringUtils.isNotBlank(study.getTrialDbId())) { + trialDbIds.add(study.getTrialDbId()); + } else { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } }); - return trialByNameNoScope; - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); + } else { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(",", missingIds)); + } + + return trialDbIds; + } + + private void processAndCacheTrial(BrAPITrial existingTrial, Program program, String trialRefSource, Map> trialByNameNoScope) { + existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); + + //get TrialId from existingTrial + Optional experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource); + if (experimentIDRef.isEmpty()) { + throw new InternalServerException("An Experiment ID was not found in any of the external references"); } + UUID experimentId = UUID.fromString(experimentIDRef.get().getReferenceID()); + + trialByNameNoScope.put( + existingTrial.getTrialName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } private Map> initializeObservationUnits(Program program, List experimentImportRows) { Map> ret = new HashMap<>(); - Map obsUnitById = new HashMap<>(); - - experimentImportRows.forEach(experimentObservation -> { - if (StringUtils.isNotBlank(experimentObservation.getObsUnitID())) { - obsUnitById.put(experimentObservation.getObsUnitID(), experimentObservation); + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } else { + rowByObsUnitId.put(row.getObsUnitID(), row); + } } }); try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(obsUnitById.keySet(), program); - - if (existingObsUnits.size() != obsUnitById.size()) { - //TODO figure out which of the OUs wasn't found + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() != rowByObsUnitId.size()) { + if(existingObsUnits.size() > rowByObsUnitId.size()) { + //in theory this shouldn't happen, but doesn't hurt to catch it in case + throw new IllegalStateException("Too many observation units were found (potential OU with duplicate ObsUnitId)"); + } else { + List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); + missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); + throw new IllegalStateException("Could not find observation units for ObsUnitId(s): " + String.join(",", missingIds)); + } } else { - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); existingObsUnits.forEach(brAPIObservationUnit -> { Optional idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource); if (idRef.isPresent()) { - ExperimentObservation experimentObservation = obsUnitById.get(idRef.get() - .getReferenceID()); - experimentObservation.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - ret.put(createObservationUnitKey(experimentObservation.getEnv(), experimentObservation.getExpUnitId()), + ExperimentObservation row = rowByObsUnitId.get(idRef.get().getReferenceID()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + ret.put(createObservationUnitKey(row), new PendingImportObject<>(ImportObjectState.EXISTING, brAPIObservationUnit, - UUID.fromString(idRef.get() - .getReferenceID()))); + UUID.fromString(idRef.get().getReferenceID()))); } else { throw new InternalServerException("An ObservationUnit ID was not found in any of the external references"); } @@ -1063,15 +1178,11 @@ private Map> initializeObserva return ret; } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } } - private String simpleStudyName(String scopedName) { - return scopedName.replaceFirst(" \\[.*\\]", ""); - } - private void validateTimeStampValue(String value, String columnHeader, ValidationErrors validationErrors, int row) { if (StringUtils.isBlank(value)) { @@ -1200,7 +1311,7 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { BrAPISeason targetSeason = null; List seasons; try { - seasons = this.brAPISeasonDAO.getSeasonByYear(year, programId); + seasons = this.brAPISeasonDAO.getSeasonsByYear(year, programId); for (BrAPISeason season : seasons) { if (null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)) { targetSeason = season; @@ -1219,12 +1330,10 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { log.error(e.getResponseBody(), e); } - String seasonDbId = (targetSeason == null) ? null : targetSeason.getSeasonDbId(); - return seasonDbId; + return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); } private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason targetSeason = null; BrAPISeason season = null; try { season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); @@ -1232,7 +1341,6 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { log.error(e.getResponseBody(), e); } Integer yearInt = (season == null) ? null : season.getYear(); - String yearStr = (yearInt == null) ? "" : yearInt.toString(); - return yearStr; + return (yearInt == null) ? "" : yearInt.toString(); } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index 820fba4ab..11d5744b7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -112,7 +112,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List createdLocations = new ArrayList<>(); try { - createdLocations.addAll(brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload)); + createdLocations.addAll(brAPILocationDAO.createBrAPILocations(locations, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 83cf4d2ae..5a8f0c907 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -192,7 +192,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List createdObservations = new ArrayList<>(); try { - createdObservations.addAll(brAPIObservationDAO.createBrAPIObservation(observations, program.getId(), upload)); + createdObservations.addAll(brAPIObservationDAO.createBrAPIObservations(observations, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index 4a041cad9..d05bf171f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -67,7 +67,7 @@ public void getExistingBrapiData(List importRows, Program program) List existingStudies; try { - existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies = brAPIStudyDAO.getStudiesByName(uniqueStudyNames, program); existingStudies.forEach(existingStudy -> { studyByName.put(existingStudy.getStudyName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); }); @@ -127,7 +127,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program // POST Study List createdStudies = new ArrayList<>(); try { - createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload)); + createdStudies.addAll(brAPIStudyDAO.createBrAPIStudies(studies, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java index 18ee6d37c..71fa6d04c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java @@ -62,7 +62,7 @@ public void getExistingBrapiData(List importRows, Program program) List existingTrials; try { - existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + existingTrials = brapiTrialDAO.getTrialsByName(uniqueTrialNames, program); existingTrials.forEach(existingTrial -> { trialByName.put(existingTrial.getTrialName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); }); @@ -114,7 +114,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List createdTrials = new ArrayList<>(); try { - createdTrials.addAll(brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload)); + createdTrials.addAll(brapiTrialDAO.createBrAPITrials(trials, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } From 6293ac2899eee37dfb60b3ea432bb5980b8a1ca4 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 26 Jan 2023 16:53:04 -0500 Subject: [PATCH 03/15] [BI-1195] Updating experiment import validation logic More refactored to the ExperimentProcessor to improve readability/maintainability --- .../importer/daos/BrAPIObservationDAO.java | 22 +- .../ExperimentObservation.java | 54 +- .../importer/services/FileImportService.java | 29 +- .../processors/ExperimentProcessor.java | 561 +++++++++--------- .../processors/GermplasmProcessor.java | 1 - .../services/processors/Processor.java | 2 +- 6 files changed, 355 insertions(+), 314 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index d2006644d..79464c03c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -33,10 +33,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -69,9 +66,22 @@ public List getObservationsByStudyName(List studyNames ); } + public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); + observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + observationSearchRequest.setObservationUnitDbIds(new ArrayList<>(ouDbIds)); + observationSearchRequest.setObservationVariableDbIds(new ArrayList<>(variableDbIds)); + ObservationsApi api = new ObservationsApi(programDAO.getCoreClient(program.getId())); + return brAPIDAOUtil.search( + api::searchObservationsPost, + (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), + observationSearchRequest + ); + } + @NotNull - private ApiResponse, Optional>> - searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { + private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); return api.searchObservationsSearchResultsDbIdGet(APPLICATION_JSON, searchResultsDbId, page, pageSize); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 722cb25fb..bf79229d9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -47,71 +47,71 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "germplasmName", name = "Germplasm Name", description = "Name of germplasm") + @ImportFieldMetadata(id = "germplasmName", name = Columns.GERMPLASM_NAME, description = "Name of germplasm") private String germplasmName; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "gid", name = "Germplasm GID", description = "Unique germplasm identifier") + @ImportFieldMetadata(id = "gid", name = Columns.GERMPLASM_GID, description = "Unique germplasm identifier") private String gid; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "test_or_check", name = "Test or Check", description = "T test (T) and check (C) germplasm") + @ImportFieldMetadata(id = "test_or_check", name = Columns.TEST_CHECK, description = "T test (T) and check (C) germplasm") private String testOrCheck; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "exp_title", name = "Experiment Title", description = "Title of experiment") + @ImportFieldMetadata(id = "exp_title", name = Columns.EXP_TITLE, description = "Title of experiment") private String expTitle; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "expDescription", name = "Experiment Description", description = "Description of experiment") + @ImportFieldMetadata(id = "expDescription", name = Columns.EXP_DESCRIPTION, description = "Description of experiment") private String expDescription; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "expUnit", name = "Experiment Unit", description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") + @ImportFieldMetadata(id = "expUnit", name = Columns.EXP_UNIT, description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") private String expUnit; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "expType", name = "Experiment Type", description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") + @ImportFieldMetadata(id = "expType", name = Columns.EXP_TYPE, description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") private String expType; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "env", name = "Environment", description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") + @ImportFieldMetadata(id = "env", name = Columns.ENV, description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") private String env; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "envLocation", name = "Environment Location", description = "Location of the environment") + @ImportFieldMetadata(id = "envLocation", name = Columns.ENV_LOCATION, description = "Location of the environment") private String envLocation; @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id = "envYear", name = "Environment Year", description = "Year corresponding to the environment") + @ImportFieldMetadata(id = "envYear", name = Columns.ENV_YEAR, description = "Year corresponding to the environment") private String envYear; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "expUnitId", name = "Experiment Unit ID", description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") + @ImportFieldMetadata(id = "expUnitId", name = Columns.EXP_UNIT_ID, description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") private String expUnitId; @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id = "expReplicateNo", name = "Experiment Replicate Number", description = "Sequential number of experimental replications") + @ImportFieldMetadata(id = "expReplicateNo", name = Columns.REP_NUM, description = "Sequential number of experimental replications") private String expReplicateNo; @ImportFieldType(type = ImportFieldTypeEnum.INTEGER) - @ImportFieldMetadata(id = "expBlockNo", name = "Experiment Block Number", description = "Sequential number of blocks in an experimental design") + @ImportFieldMetadata(id = "expBlockNo", name = Columns.BLOCK_NUM, description = "Sequential number of blocks in an experimental design") private String expBlockNo; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "row", name = "Row", description = "Horizontal (y-axis) position in 2D Cartesian space.") + @ImportFieldMetadata(id = "row", name = Columns.ROW, description = "Horizontal (y-axis) position in 2D Cartesian space.") private String row; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "column", name = "Column", description = "Vertical (x-axis) position in 2D Cartesian space.") + @ImportFieldMetadata(id = "column", name = Columns.COLUMN, description = "Vertical (x-axis) position in 2D Cartesian space.") private String column; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "treatmentFactors", name = "Treatment Factors", description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") + @ImportFieldMetadata(id = "treatmentFactors", name = Columns.TREATMENT_FACTORS, description = "Treatment factors in an experiment with applied variables, like fertilizer or water regimens.") private String treatmentFactors; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "obsUnitID", name = "Observation Unit ID", description = "A database generated unique identifier for experimental observation units") + @ImportFieldMetadata(id = "obsUnitID", name = Columns.OBS_UNIT_ID, description = "A database generated unique identifier for experimental observation units") private String obsUnitID; public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit, String referenceSource, UUID id, String expSeqValue) { @@ -349,4 +349,24 @@ private void addReference(List refs, UUID uuid, String r refs.add(reference); } + public static final class Columns { + public static final String GERMPLASM_NAME = "Germplasm Name"; + public static final String GERMPLASM_GID = "Germplasm GID"; + public static final String TEST_CHECK = "Test (T) or Check (C)"; + public static final String EXP_TITLE = "Exp Title"; + public static final String EXP_DESCRIPTION = "Exp Description"; + public static final String EXP_UNIT = "Exp Unit"; + public static final String EXP_TYPE = "Exp Type"; + public static final String ENV = "Env"; + public static final String ENV_LOCATION = "Env Location"; + public static final String ENV_YEAR = "Env Year"; + public static final String EXP_UNIT_ID = "Exp Unit ID"; + public static final String REP_NUM = "Exp Replicate #"; + public static final String BLOCK_NUM = "Exp Block #"; + public static final String ROW = "Row"; + public static final String COLUMN = "Column"; + public static final String TREATMENT_FACTORS = "Treatment Factors"; + public static final String OBS_UNIT_ID = "ObsUnitID"; + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 8f7f3f549..19bdbd1cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -28,6 +28,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.tika.mime.MediaType; import org.brapi.client.v2.JSON; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; @@ -51,6 +52,7 @@ import org.breedinginsight.services.parsers.MimeTypeParser; import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.jooq.JSONB; import tech.tablesaw.api.Table; @@ -328,11 +330,8 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU User user = userService.getById(actingUser.getId()).get(); // Find the import - Optional uploadOptional = importDAO.getUploadById(uploadId); - if (uploadOptional.isEmpty()) { - throw new DoesNotExistException("Upload with that id does not exist"); - } - ImportUpload upload = uploadOptional.get(); + ImportUpload upload = importDAO.getUploadById(uploadId) + .orElseThrow(() -> new DoesNotExistException("Upload with that id does not exist")); if (upload.getProgress() != null && upload.getProgress().getStatuscode().equals((short) HttpStatus.ACCEPTED.getCode())) { // Another action is in process for this import, throw an error @@ -345,17 +344,11 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } // Get mapping - Optional mappingConfigOptional = importMappingDAO.getMapping(upload.getImporterMappingId()); - if (mappingConfigOptional.isEmpty()) { - throw new DoesNotExistException("Cannot find mapping config associated with upload."); - } - ImportMapping mappingConfig = mappingConfigOptional.get(); + ImportMapping mappingConfig = importMappingDAO.getMapping(upload.getImporterMappingId()) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); - Optional optionalImportService = configManager.getImportServiceById(mappingConfig.getImportTypeId()); - if (optionalImportService.isEmpty()) { - throw new DoesNotExistException("Config with that id does not exist"); - } - BrAPIImportService importService = optionalImportService.get(); + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); // TODO: maybe return brapiimport from configmanager // Get our data @@ -469,7 +462,11 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); } catch (Exception e) { - log.error(e.getMessage(), e); + if(e instanceof ApiException) { + log.error("Error making BrAPI call: " + Utilities.generateApiExceptionLogMessage((ApiException) e), e); + } else { + log.error(e.getMessage(), e); + } ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode()); // TODO: Probably don't want to return this message. But do it for now diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 21215ec71..fe4929378 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -80,10 +81,17 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.\n" + + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + "for each experiment unit)."; + private static final String MIDNIGHT = "T00:00:00-00:00"; + private static final String TIMESTAMP_PREFIX = "TS:"; + private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + private static final String COMMA_DELIMITER = ","; + private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; + private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; + private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -186,7 +194,7 @@ public Map process( Table data, Program program, User user, - boolean commit) throws ValidatorException, MissingRequiredInfoException { + boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { ValidationErrors validationErrors = new ValidationErrors(); @@ -196,33 +204,33 @@ public Map process( List> timestampCols = new ArrayList<>(); for (Column dynamicCol : dynamicCols) { //Distinguish between phenotype and timestamp columns - if (dynamicCol.name().startsWith("TS:")) { + if (dynamicCol.name().startsWith(TIMESTAMP_PREFIX)) { timestampCols.add(dynamicCol); } else { phenotypeCols.add(dynamicCol); } } - validateObservationAndTimestampColumns(program.getId(), phenotypeCols, timestampCols, validationErrors); + List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols, validationErrors); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval for (Column tsColumn : timestampCols) { - timeStampColByPheno.put(tsColumn.name().replaceFirst("^TS:\\s*", ""), tsColumn); + timeStampColByPheno.put(tsColumn.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), tsColumn); } // add "New" pending data to the BrapiData objects initNewBrapiData(importRows, phenotypeCols, program, user, commit); - prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport, validationErrors); + prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); - validateFields(importRows, validationErrors); + validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols); if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); } // Construct our response object - return getStatisticsMap(importRows); + return generateStatisticsMap(importRows); } @Override @@ -294,11 +302,11 @@ public void postBrapiData(Map mappedBrAPIImport, Program } } - private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport, ValidationErrors validationErrors) throws MissingRequiredInfoException { - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) throws MissingRequiredInfoException { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); mappedImportRow.setTrial(this.trialByNameNoScope.get(importRow.getExpTitle())); mappedImportRow.setLocation(this.locationByName.get(importRow.getEnvLocation())); mappedImportRow.setStudy(this.studyByNameNoScope.get(importRow.getEnv())); @@ -315,21 +323,11 @@ private void prepareDataForValidation(List importRows, List germplasmPIO = getGidPOI(importRow); mappedImportRow.setGermplasm(germplasmPIO); - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, i, germplasmPIO); - } - - //Check if existing environment. If so, ObsUnitId must be assigned - if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) - && (StringUtils.isBlank(importRow.getObsUnitID()))) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } - - mappedBrAPIImport.put(i, mappedImportRow); + mappedBrAPIImport.put(rowNum, mappedImportRow); } } - private void validateObservationAndTimestampColumns(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { + private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { Set varNames = phenotypeCols.stream() .map(Column::name) .collect(Collectors.toSet()); @@ -350,38 +348,19 @@ private void validateObservationAndTimestampColumns(UUID programId, List unmatchedTimestamps = tsNames.stream() - .filter(e -> !(varNames.contains(e.replaceFirst("^TS:\\s*", "")))) - .collect(Collectors.toList()); - if (unmatchedTimestamps.size() > 0) { - //TODO convert this to a ValidationError - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(", ", unmatchedTimestamps)); - } - - // Perform ontology validations on each observation value in phenotype column - Map colVarMap = filteredTraits.stream() - .collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); - - for (Column column : phenotypeCols) { - for (int i = 0; i < column.size(); i++) { - String value = column.getString(i); - String colName = column.name(); - validateObservationValue(colVarMap.get(colName), value, colName, validationErrors, i); + "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); + } else { + // Check that each ts column corresponds to a phenotype column + List unmatchedTimestamps = tsNames.stream() + .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .collect(Collectors.toList()); + if (unmatchedTimestamps.size() > 0) { + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); } - } - //Timestamp validation - for (Column column : timestampCols) { - for (int i = 0; i < column.size(); i++) { - String value = column.getString(i); - String colName = column.name(); - validateTimeStampValue(value, colName, validationErrors, i); - } + return filteredTraits; } } @@ -419,8 +398,8 @@ private void initNewBrapiData(List importRows, List> phen } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); PendingImportObject trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); @@ -449,17 +428,16 @@ private void initNewBrapiData(List importRows, List> phen for (Column column : phenotypeCols) { //If associated timestamp column, add String dateTimeValue = null; - if (timeStampColByPheno.get(column.name()) != null) { - dateTimeValue = timeStampColByPheno.get(column.name()) - .getString(i); + if (timeStampColByPheno.containsKey(column.name())) { + dateTimeValue = timeStampColByPheno.get(column.name()).getString(rowNum); //If no timestamp, set to midnight if (!dateTimeValue.isBlank() && !validDateTimeValue(dateTimeValue)) { - dateTimeValue += "T00:00:00-00:00"; + dateTimeValue += MIDNIGHT; } } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - fetchOrCreateObservationPIO(importRow, column.name(), column.getString(i), dateTimeValue, commit, seasonDbId, obsUnitPIO); + fetchOrCreateObservationPIO(importRow, column.name(), column.getString(rowNum), dateTimeValue, commit, seasonDbId, obsUnitPIO); } } } @@ -483,100 +461,166 @@ private String getObservationHash(String observationUnitName, String variableNam return DigestUtils.sha256Hex(concat); } - private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { - HashSet uniqueStudyAndObsUnit = new HashSet<>(); - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - validateConditionallyRequired(validationErrors, i, importRow); - validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, i, importRow); + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols) throws MissingRequiredInfoException, ApiException { + //fetching any existing observations for any OUs in the import + Map existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + Map colVarMap = referencedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); + + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + + //Check if existing environment. If so, ObsUnitId must be assigned + if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) + && (StringUtils.isBlank(importRow.getObsUnitID()))) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } + + validateConditionallyRequired(validationErrors, rowNum, importRow); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash); } return validationErrors; } + private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + } + + private Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, BrAPIStudy::getStudyName)); + + observationUnitByNameNoScope.values().forEach(ou -> { + if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { + ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), ou.getBrAPIObject()); + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()).getObservationVariableName(); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()).getObservationUnitName(); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, Map colVarMap, Map existingObservations) { + phenotypeCols.forEach(phenoCol -> { + if(existingObservations.containsKey(getImportObservationHash(importRow, phenoCol.name()))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if(timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + }); + } + /** - * Validate that the the observation unit is unique within a study. + * Validate that the observation unit is unique within a study. *
* SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. * * @param validationErrors can be modified as a side effect. * @param uniqueStudyAndObsUnit can be modified as a side effect. - * @param i counter that is always two less the file row being validated + * @param rowNum counter that is always two less the file row being validated * @param importRow the data row being validated */ private void validateUniqueObsUnits( ValidationErrors validationErrors, - HashSet uniqueStudyAndObsUnit, - int i, + Set uniqueStudyAndObsUnit, + int rowNum, ExperimentObservation importRow) { String envIdPlusStudyId = createObservationUnitKey(importRow); if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); - this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); + this.addRowError(Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); } else { uniqueStudyAndObsUnit.add(envIdPlusStudyId); } } - private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { - String experimentTitle = importRow.getExpTitle(); - String obsUnitID = importRow.getObsUnitID(); - if (StringUtils.isBlank(obsUnitID)) { - validateRequiredCell( - experimentTitle, - "Exp Title", - "Field is blank", validationErrors, i - ); + private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; } - ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle) - .getState(); - boolean isExperimentNew = (expState == ImportObjectState.NEW); - - if (isExperimentNew) { - String errorMessage = "Field is blank when creating a new experiment"; - validateRequiredCell(importRow.getGid(), - "GID", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getExpUnit(), - "Exp Unit", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getExpType(), - "Exp Type", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getEnv(), - "Env", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getEnvLocation(), - "Env Location", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getEnvYear(), - "Env Year", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getExpUnitId(), - "Exp Unit ID", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getExpReplicateNo(), - "Exp Replicate #", - errorMessage, validationErrors, i); - validateRequiredCell(importRow.getExpBlockNo(), - "Exp Block #", - errorMessage, validationErrors, i); + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(),Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), Columns.ENV, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnvLocation(), Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnitId(), Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); } } - private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int i) { + private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int rowNum) { if (StringUtils.isBlank(value)) { - addRowError( - columnHeader, - errorMessage, - validationErrors, i - ); + addRowError(columnHeader, errorMessage, validationErrors, rowNum); } } - private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int i) { + private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int rowNum) { ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + validationErrors.addError(rowNum + 2, ve); // +2 instead of +1 to account for the column header row. } private void addIfNotNull(HashSet set, String setValue) { @@ -585,7 +629,7 @@ private void addIfNotNull(HashSet set, String setValue) { } } - private Map getStatisticsMap(List importRows) { + private Map generateStatisticsMap(List importRows) { // Data for stats. HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's @@ -629,14 +673,10 @@ private Map getStatisticsMap(List ); } - private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { + private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum, PendingImportObject germplasmPIO) { // error if GID is not blank but GID does not already exist if (StringUtils.isNotBlank(importRow.getGid()) && germplasmPIO == null) { - addRowError( - "GID", - "A non-existing GID", - validationErrors, i - ); + addRowError(Columns.GERMPLASM_GID, "A non-existing GID", validationErrors, rowNum); } } @@ -761,7 +801,7 @@ private void updateObservationDependencyValues(Program program) { // Update ObservationVariable DbIds List traits = getTraitList(program); - Map traitMap = traits.stream().collect(Collectors.toMap(TraitEntity::getObservationVariableName, trait -> trait)); + Map traitMap = traits.stream().collect(Collectors.toMap(TraitEntity::getObservationVariableName, Function.identity())); for (PendingImportObject observation : this.observationByHash.values()) { String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); @@ -913,34 +953,69 @@ private ArrayList getGermplasmByAccessionNumber( return resultGermplasm; } - private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { - Map> existingGermplasmByGID = new HashMap<>(); + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> ret = new HashMap<>(); - List existingGermplasms; - if(observationUnitByNameNoScope.size() > 0) { - Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - try { - existingGermplasms = brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId()); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } else { + rowByObsUnitId.put(row.getObsUnitID(), row); + } } - } else { - List uniqueGermplasmGIDs = experimentImportRows.stream() - .map(ExperimentObservation::getGid) - .distinct() - .collect(Collectors.toList()); + }); - try { - existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); - } catch (ApiException e) { - log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + Optional idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource); + if (idRef.isPresent()) { + ExperimentObservation row = rowByObsUnitId.get(idRef.get().getReferenceID()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + ret.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.get().getReferenceID()))); + } else { + throw new InternalServerException("An ObservationUnit ID was not found in any of the external references"); + } + }); } + + return ret; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); } + } - existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); - return existingGermplasmByGID; + private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { + Map> trialByName = new HashMap<>(); + String programKey = program.getKey(); + + initializeTrialsForExistingObservationUnits(program, experimentImportRows, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey, null)) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + try { + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); + brapiTrialDAO.getTrialsByName(uniqueTrialNames, program) + .forEach(existingTrial -> processAndCacheTrial(existingTrial, program, trialRefSource, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; } private Map> initializeStudyByNameNoScope(Program program, List experimentImportRows) { @@ -950,106 +1025,96 @@ private Map> initializeStudyByNameNoScop } List existingStudies; - if(observationUnitByNameNoScope.size() > 0) { - List studyDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getStudyDbId()).collect(Collectors.toList()); - try { - existingStudies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - if(existingStudies.size() < studyDbIds.size()) { - throw new IllegalStateException("Did not find all of the studies for the requested list of studyDbIds"); - } - } else { - //TODO rethink this somewhat...should there be support for both fetching studies from existing OUs and new OUs? - ExperimentObservation experimentObservation = experimentImportRows.get(0); - - PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); - - UUID experimentId = trial.getId(); - + Optional expTitle = experimentImportRows.stream().filter(row -> StringUtils.isBlank(row.getObsUnitID())).map(ExperimentObservation::getExpTitle).findFirst(); + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + PendingImportObject trial = this.trialByNameNoScope.get(expTitle.get()); + UUID experimentId = trial.getId(); - try { - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } + try { + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); } existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); return studyByName; } - private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { - //Swap season DbId with year String - String seasonId = existingStudy.getSeasons().get(0); - existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); - - existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); - studyByName.put( - existingStudy.getStudyName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); - } - private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { Map> locationByName = new HashMap<>(); - List existingLocations; + List existingLocations = new ArrayList<>(); if(studyByNameNoScope.size() > 0) { Set locationDbIds = studyByNameNoScope.values().stream().map(study -> study.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); try { - existingLocations = brAPILocationDAO.getLocationsByDbId(locationDbIds, program.getId()); + existingLocations.addAll(brAPILocationDAO.getLocationsByDbId(locationDbIds, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } - } else { - List uniqueLocationNames = experimentImportRows.stream() - .map(ExperimentObservation::getEnvLocation) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); + } + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isNotBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); - try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); - } catch (ApiException e) { - log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } + try { + existingLocations.addAll(brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); } - existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); - }); + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation))); return locationByName; } - private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { - Map> trialByName = new HashMap<>(); - String programKey = program.getKey(); + private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); - initializeTrialsForExistingObservationUnits(program, experimentImportRows, trialByName); + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isNotBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isNotBlank(row.getObsUnitID())) - .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey, null)) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); try { - String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - brapiTrialDAO.getTrialsByName(uniqueTrialNames, program) - .forEach(existingTrial -> processAndCacheTrial(existingTrial, program, trialRefSource, trialByName)); + existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } - return trialByName; + existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); + return existingGermplasmByGID; + } + + private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { + //Swap season DbId with year String + String seasonId = existingStudy.getSeasons().get(0); + existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); + + existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); + studyByName.put( + existingStudy.getStudyName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); } private void initializeTrialsForExistingObservationUnits(Program program, List experimentImportRows, Map> trialByName) { @@ -1089,7 +1154,7 @@ private void initializeTrialsForExistingObservationUnits(Program program, List missingIds = new ArrayList<>(trialDbIds); missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); - throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(",", missingIds)); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); } } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); @@ -1112,7 +1177,7 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr } else { List missingIds = new ArrayList<>(trialDbIds); missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(",", missingIds)); + throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); } return trialDbIds; @@ -1133,56 +1198,6 @@ private void processAndCacheTrial(BrAPITrial existingTrial, Program program, Str new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } - private Map> initializeObservationUnits(Program program, List experimentImportRows) { - Map> ret = new HashMap<>(); - - Map rowByObsUnitId = new HashMap<>(); - experimentImportRows.forEach(row -> { - if (StringUtils.isNotBlank(row.getObsUnitID())) { - if(rowByObsUnitId.containsKey(row.getObsUnitID())) { - throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } else { - rowByObsUnitId.put(row.getObsUnitID(), row); - } - } - }); - - try { - List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (existingObsUnits.size() != rowByObsUnitId.size()) { - if(existingObsUnits.size() > rowByObsUnitId.size()) { - //in theory this shouldn't happen, but doesn't hurt to catch it in case - throw new IllegalStateException("Too many observation units were found (potential OU with duplicate ObsUnitId)"); - } else { - List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); - missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); - throw new IllegalStateException("Could not find observation units for ObsUnitId(s): " + String.join(",", missingIds)); - } - } else { - existingObsUnits.forEach(brAPIObservationUnit -> { - Optional idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource); - if (idRef.isPresent()) { - ExperimentObservation row = rowByObsUnitId.get(idRef.get().getReferenceID()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - ret.put(createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.get().getReferenceID()))); - } else { - throw new InternalServerException("An ObservationUnit ID was not found in any of the external references"); - } - }); - } - - return ret; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - private void validateTimeStampValue(String value, String columnHeader, ValidationErrors validationErrors, int row) { if (StringUtils.isBlank(value)) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 5a2709b28..7276a36ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -92,7 +92,6 @@ public class GermplasmProcessor implements Processor { @Inject public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContext dsl, BreedingMethodDAO breedingMethodDAO, BrAPIListDAO brAPIListDAO) { - this.brAPIGermplasmService = brAPIGermplasmService; this.dsl = dsl; this.breedingMethodDAO = breedingMethodDAO; this.brAPIListDAO = brAPIListDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 9c72ea4f4..26ba7a1ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -53,7 +53,7 @@ public interface Processor { Map process(List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) - throws ValidatorException, MissingRequiredInfoException; + throws ValidatorException, MissingRequiredInfoException, ApiException; /** * Given mapped brapi import with updates from prior dependencies, check if have everything needed From 98b1fe9112d045b3577f99ee9434acebe2c53388 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 6 Feb 2023 23:50:16 -0500 Subject: [PATCH 04/15] [BI-1195] Implementing unit test for experiment import. Fixing bugs and adding methods as needed to get the tests to pass --- .../api/v1/controller/ProgramController.java | 18 +- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 2 +- .../daos/BrAPIObservationUnitDAO.java | 12 + .../brapps/importer/model/base/Location.java | 7 +- .../importer/model/imports/PendingImport.java | 3 +- .../ExperimentImportService.java | 1 + .../ExperimentObservation.java | 28 +- .../processors/ExperimentProcessor.java | 113 ++- .../processors/LocationProcessor.java | 46 +- .../services/processors/StudyProcessor.java | 5 +- .../daos/ProgramLocationDAO.java | 194 +++-- .../daos/impl/TraitDAOImpl.java | 7 +- .../breedinginsight/model/BrAPIConstants.java | 2 +- .../model/ProgramLocation.java | 1 + .../services/ProgramLocationService.java | 74 +- .../services/ProgramService.java | 15 + .../exceptions/ValidatorException.java | 2 + .../utilities/BrAPIDAOUtil.java | 28 +- .../breedinginsight/utilities/Utilities.java | 4 +- .../importer/ExperimentFileImportTest.java | 755 ++++++++++++++++++ ...eMap.java => GermplasmFileImportTest.java} | 217 ++--- .../brapps/importer/ImportTestUtils.java | 162 ++++ .../daos/BrAPIDAOUtilUnitTest.java | 4 +- 23 files changed, 1363 insertions(+), 337 deletions(-) create mode 100644 src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java rename src/test/java/org/breedinginsight/brapps/importer/{GermplasmTemplateMap.java => GermplasmFileImportTest.java} (80%) create mode 100644 src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java b/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java index 43c24aa52..39b046cd5 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ProgramController.java @@ -24,6 +24,7 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.*; import org.breedinginsight.api.model.v1.request.*; import org.breedinginsight.api.model.v1.request.query.QueryParams; @@ -37,6 +38,7 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.ProgramObservationLevelService; import org.breedinginsight.services.exceptions.*; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.mappers.ProgramLocationQueryMapper; import org.breedinginsight.utilities.response.mappers.ProgramQueryMapper; import org.breedinginsight.api.model.v1.response.DataResponse; @@ -302,6 +304,9 @@ public HttpResponse>> getProgramLocations } catch (DoesNotExistException e){ log.info(e.getMessage()); return HttpResponse.notFound(); + } catch (ApiException e) { + log.error("Error fetching program locations: " + Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); } } @@ -318,8 +323,11 @@ public HttpResponse>> postProgramLocation return ResponseUtils.getQueryResponse(programLocations, programLocationQueryMapper, searchRequest, queryParams); } catch (DoesNotExistException e){ - log.info(e.getMessage()); + log.error(e.getMessage(), e); return HttpResponse.notFound(); + } catch (ApiException e) { + log.error("Error fetching program locations: " + Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); } } @@ -332,7 +340,13 @@ public HttpResponse>> postProgramLocation public HttpResponse> getProgramLocations(@PathVariable UUID programId, @PathVariable UUID locationId) { - Optional programLocation = programLocationService.getById(programId, locationId); + Optional programLocation = null; + try { + programLocation = programLocationService.getById(programId, locationId); + } catch (ApiException e) { + log.error("Error fetching program location: " + Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } if(programLocation.isPresent()) { Response response = new Response(programLocation.get()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index f18d730d2..1f5c83e18 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -283,7 +283,7 @@ public Optional getGermplasmByDBID(String germplasmDbId, UUID pr return Optional.ofNullable(germplasm); } - public List getGermplasmsByDBID(Set germplasmDbIds, UUID programId) throws ApiException { + public List getGermplasmsByDBID(Collection germplasmDbIds, UUID programId) throws ApiException { Map cache = programGermplasmCache.get(programId); //key is UUID, want to filter by DBID List germplasm = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index 39aaa3a11..a83a4c9d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -102,4 +102,16 @@ public List getObservationUnitsById(Collection obs api::searchObservationunitsSearchResultsDbIdGet, observationUnitSearchRequest); } + + public List getObservationUnitsForStudyDbId(String studyDbId, Program program) throws ApiException { + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); + observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() + .getProgramDbId())); + observationUnitSearchRequest.studyDbIds(List.of(studyDbId)); + + ObservationUnitsApi api = new ObservationUnitsApi(programDAO.getCoreClient(program.getId())); + return brAPIDAOUtil.search(api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + observationUnitSearchRequest); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java index 547147412..5df11586e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java @@ -24,6 +24,7 @@ import org.breedinginsight.brapps.importer.model.config.ImportFieldType; import org.breedinginsight.brapps.importer.model.config.ImportFieldTypeEnum; import org.breedinginsight.brapps.importer.model.config.ImportMappingRequired; +import org.breedinginsight.model.ProgramLocation; @Getter @Setter @@ -36,9 +37,9 @@ public class Location implements BrAPIObject { @ImportFieldMetadata(id="locationName", name="Location Name", description = "The name of the location.") private String locationName; - public BrAPILocation constructBrAPILocation() { - BrAPILocation location = new BrAPILocation(); - location.setLocationName(getLocationName()); + public ProgramLocation constructLocation() { + ProgramLocation location = new ProgramLocation(); + location.setName(getLocationName()); return location; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index 42b4699b2..f3d4f3180 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -28,6 +28,7 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; import java.util.ArrayList; import java.util.List; @@ -39,7 +40,7 @@ public class PendingImport { private PendingImportObject germplasm; private PendingImportObject trial; - private PendingImportObject location; + private PendingImportObject location; private PendingImportObject study; private PendingImportObject observationUnit; private List> observations = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 19c4a3c22..b12f8ec40 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index bf79229d9..fa6cc749f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -24,12 +24,12 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; @@ -111,7 +111,7 @@ public class ExperimentObservation implements BrAPIImport { private String treatmentFactors; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "obsUnitID", name = Columns.OBS_UNIT_ID, description = "A database generated unique identifier for experimental observation units") + @ImportFieldMetadata(id = "ObsUnitID", name = Columns.OBS_UNIT_ID, description = "A database generated unique identifier for experimental observation units") private String obsUnitID; public BrAPITrial constructBrAPITrial(Program program, User user, boolean commit, String referenceSource, UUID id, String expSeqValue) { @@ -152,9 +152,9 @@ private void setBrAPITrialCommitFields(Program program, BrAPITrial trial, String } - public BrAPILocation constructBrAPILocation() { - BrAPILocation location = new BrAPILocation(); - location.setLocationName(getEnvLocation()); + public ProgramLocation constructProgramLocation() { + ProgramLocation location = new ProgramLocation(); + location.setName(getEnvLocation()); return location; } @@ -191,10 +191,8 @@ public BrAPIStudy constructBrAPIStudy( design.setPUI(designType); design.setDescription(designType); study.setExperimentalDesign(design); - String envSequenceValue = null; if (commit) { - envSequenceValue = envNextVal.get().toString(); - study.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envSequenceValue); + study.putAdditionalInfoItem(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER, envNextVal.get().toString()); } return study; } @@ -244,10 +242,10 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( // Block number if (getExpBlockNo() != null) { - BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); - repLvl.setLevelName(BrAPIConstants.BLOCK.getValue()); - repLvl.setLevelCode(getExpBlockNo()); - levelRelationships.add(repLvl); + BrAPIObservationUnitLevelRelationship blockLvl = new BrAPIObservationUnitLevelRelationship(); + blockLvl.setLevelName(BrAPIConstants.BLOCK.getValue()); + blockLvl.setLevelCode(getExpBlockNo()); + levelRelationships.add(blockLvl); } position.setObservationLevelRelationships(levelRelationships); @@ -265,7 +263,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } if (getColumn() != null) { position.setPositionCoordinateY(getColumn()); - position.setPositionCoordinateYType(BrAPIPositionCoordinateTypeEnum.GRID_ROW); + position.setPositionCoordinateYType(BrAPIPositionCoordinateTypeEnum.GRID_COL); } observationUnit.setObservationUnitPosition(position); @@ -337,7 +335,7 @@ private List getStudyExternalReferences( private List getObsUnitExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { - return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, obsUnitId); } @@ -352,7 +350,7 @@ private void addReference(List refs, UUID uuid, String r public static final class Columns { public static final String GERMPLASM_NAME = "Germplasm Name"; public static final String GERMPLASM_GID = "Germplasm GID"; - public static final String TEST_CHECK = "Test (T) or Check (C)"; + public static final String TEST_CHECK = "Test (T) or Check (C )"; public static final String EXP_TITLE = "Exp Title"; public static final String EXP_DESCRIPTION = "Exp Description"; public static final String EXP_UNIT = "Exp Unit"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index fe4929378..6abda1bd9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -26,7 +26,6 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; @@ -34,6 +33,8 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -50,11 +51,9 @@ import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.Scale; -import org.breedinginsight.model.Trait; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; @@ -98,7 +97,7 @@ public class ExperimentProcessor implements Processor { private final DSLContext dsl; private final BrAPITrialDAO brapiTrialDAO; - private final BrAPILocationDAO brAPILocationDAO; + private final ProgramLocationService locationService; private final BrAPIStudyDAO brAPIStudyDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPIObservationDAO brAPIObservationDAO; @@ -115,7 +114,7 @@ public class ExperimentProcessor implements Processor { //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; - private Map> locationByName = null; + private Map> locationByName = null; private Map> studyByNameNoScope = null; // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) @@ -132,7 +131,7 @@ public class ExperimentProcessor implements Processor { @Inject public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, - BrAPILocationDAO brAPILocationDAO, + ProgramLocationService locationService, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, BrAPIObservationDAO brAPIObservationDAO, @@ -142,7 +141,7 @@ public ExperimentProcessor(DSLContext dsl, FileMappingUtil fileMappingUtil) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; - this.brAPILocationDAO = brAPILocationDAO; + this.locationService = locationService; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.brAPIObservationDAO = brAPIObservationDAO; @@ -195,6 +194,7 @@ public Map process( Program program, User user, boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { + log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -229,6 +229,7 @@ public Map process( throw new ValidatorException(validationErrors); } + log.debug("done processing experiment import"); // Construct our response object return generateStatisticsMap(importRows); } @@ -240,9 +241,15 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + log.debug("starting post of experiment data to BrAPI server"); List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); - List newLocations = ProcessorData.getNewObjects(this.locationByName); + List newLocations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); // filter out observations with no 'value' so they will not be saved @@ -250,6 +257,9 @@ public void postBrapiData(Map mappedBrAPIImport, Program .stream() .filter(obs -> !obs.getValue().isBlank()) .collect(Collectors.toList()); + + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + try { List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial @@ -260,10 +270,10 @@ public void postBrapiData(Map mappedBrAPIImport, Program .setTrialDbId(createdTrial.getTrialDbId()); } - List createdLocations = new ArrayList<>(brAPILocationDAO.createBrAPILocations(newLocations, program.getId(), upload)); + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); // set the DbId to the for each newly created trial - for (BrAPILocation createdLocation : createdLocations) { - String createdLocationName = createdLocation.getLocationName(); + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); this.locationByName.get(createdLocationName) .getBrAPIObject() .setLocationDbId(createdLocation.getLocationDbId()); @@ -296,9 +306,13 @@ public void postBrapiData(Map mappedBrAPIImport, Program updateObservationDependencyValues(program); brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + log.debug("experiment import complete"); } catch (ApiException e) { - log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e)); + log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error saving experiment import", e); + throw new InternalServerException(e.getMessage(), e); } } @@ -706,7 +720,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); this.observationUnitByNameNoScope.put(key, pio); } return pio; @@ -747,6 +761,7 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies if (commit) { String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season @@ -760,13 +775,13 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b return pio; } - private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { - PendingImportObject pio; + private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio; if (locationByName.containsKey((importRow.getEnvLocation()))) { pio = locationByName.get(importRow.getEnvLocation()); } else { - BrAPILocation newLocation = importRow.constructBrAPILocation(); - pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + ProgramLocation newLocation = importRow.constructProgramLocation(); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); this.locationByName.put(importRow.getEnvLocation(), pio); } return pio; @@ -794,8 +809,6 @@ private void updateObservationDependencyValues(Program program) { // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds this.observationUnitByNameNoScope.values().stream() - .filter(Objects::nonNull) - .distinct() .map(PendingImportObject::getBrAPIObject) .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); @@ -901,9 +914,6 @@ private void updateStudyDependencyValues(Map mappedBrAPI mappedBrAPIImport.values() .stream() .map(PendingImport::getLocation) - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) .forEach(this::updateStudyLocationDbId); // update trial DbIds in studies for all distinct trials @@ -912,17 +922,17 @@ private void updateStudyDependencyValues(Map mappedBrAPI .filter(Objects::nonNull) .distinct() .map(PendingImportObject::getBrAPIObject) - .forEach(trait -> this.updateTrialDbId(trait, programKey)); + .forEach(trial -> this.updateTrialDbId(trial, programKey)); } - private void updateStudyLocationDbId(BrAPILocation location) { + private void updateStudyLocationDbId(PendingImportObject location) { this.studyByNameNoScope.values() .stream() - .filter(study -> location.getLocationName() + .filter(study -> location.getId().toString() .equals(study.getBrAPIObject() - .getLocationName())) + .getLocationDbId())) .forEach(study -> study.getBrAPIObject() - .setLocationDbId(location.getLocationDbId())); + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); } private void updateTrialDbId(BrAPITrial trial, String programKey) { @@ -1002,9 +1012,8 @@ private Map> initializeTrialByNameNoScop List uniqueTrialNames = experimentImportRows.stream() .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey, null)) + .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey)) .distinct() - .filter(Objects::nonNull) .collect(Collectors.toList()); try { String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); @@ -1025,7 +1034,10 @@ private Map> initializeStudyByNameNoScop } List existingStudies; - Optional expTitle = experimentImportRows.stream().filter(row -> StringUtils.isBlank(row.getObsUnitID())).map(ExperimentObservation::getExpTitle).findFirst(); + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); if(expTitle.isEmpty()) { expTitle = trialByNameNoScope.keySet().stream().findFirst(); } @@ -1043,14 +1055,18 @@ private Map> initializeStudyByNameNoScop return studyByName; } - private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { - Map> locationByName = new HashMap<>(); + private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); - List existingLocations = new ArrayList<>(); + List existingLocations = new ArrayList<>(); if(studyByNameNoScope.size() > 0) { - Set locationDbIds = studyByNameNoScope.values().stream().map(study -> study.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); try { - existingLocations.addAll(brAPILocationDAO.getLocationsByDbId(locationDbIds, program.getId())); + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1058,20 +1074,20 @@ private Map> initializeUniqueLocation } List uniqueLocationNames = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isNotBlank(experimentObservation.getObsUnitID())) + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) .map(ExperimentObservation::getEnvLocation) .distinct() .filter(Objects::nonNull) .collect(Collectors.toList()); try { - existingLocations.addAll(brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId())); + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } - existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation))); + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); return locationByName; } @@ -1090,7 +1106,7 @@ private Map> initializeExistingGermp } List uniqueGermplasmGIDs = experimentImportRows.stream() - .filter(experimentObservation -> StringUtils.isNotBlank(experimentObservation.getObsUnitID())) + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) .map(ExperimentObservation::getGid) .distinct() .collect(Collectors.toList()); @@ -1102,7 +1118,13 @@ private Map> initializeExistingGermp throw new InternalServerException(e.toString(), e); } - existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); + existingGermplasms.forEach(existingGermplasm -> { + Optional xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)); + if(xref.isEmpty()) { + throw new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId()); + } + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.get().getReferenceID()))); + }); return existingGermplasmByGID; } @@ -1112,9 +1134,14 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map existingStudy.setSeasons(List.of(this.seasonDbIdToYear(seasonId, program.getId()))); existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); + + Optional xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())); + if(xref.isEmpty()) { + throw new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId()); + } studyByName.put( existingStudy.getStudyName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.get().getReferenceID()))); } private void initializeTrialsForExistingObservationUnits(Program program, List experimentImportRows, Map> trialByName) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index 11d5744b7..16b302606 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -18,8 +18,11 @@ import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPILocation; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Location; @@ -29,8 +32,11 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.api.Table; import javax.inject.Inject; @@ -41,16 +47,17 @@ import java.util.stream.Collectors; @Prototype +@Slf4j public class LocationProcessor implements Processor { private static final String NAME = "Location"; - private BrAPILocationDAO brAPILocationDAO; - private Map> locationByName = new HashMap<>(); + private ProgramLocationService locationService; + private Map> locationByName = new HashMap<>(); @Inject - public LocationProcessor(BrAPILocationDAO brAPILocationDAO) { - this.brAPILocationDAO = brAPILocationDAO; + public LocationProcessor(ProgramLocationService locationService) { + this.locationService = locationService; } public void getExistingBrapiData(List importRows, Program program) { @@ -59,12 +66,12 @@ public void getExistingBrapiData(List importRows, Program program) .map(locationImport -> locationImport.getLocation().getLocationName()) .distinct() .collect(Collectors.toList()); - List existingLocations; + List existingLocations; try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + existingLocations = locationService.getLocationsByName(uniqueLocationNames, program.getId()); existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); }); } catch (ApiException e) { // We shouldn't get an error back from our services. If we do, nothing the user can do about it @@ -84,9 +91,9 @@ public Map process(List importRows Location location = brapiImport.getLocation(); - BrAPILocation brapiLocation = location.constructBrAPILocation(); + ProgramLocation brapiLocation = location.constructLocation(); if (!locationByName.containsKey(location.getLocationName())) { - locationByName.put(brapiLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.NEW, brapiLocation)); + locationByName.put(brapiLocation.getName(), new PendingImportObject<>(ImportObjectState.NEW, brapiLocation)); mappedImportRow.setLocation(new PendingImportObject<>(ImportObjectState.NEW, brapiLocation)); } mappedImportRow.setLocation(locationByName.get(location.getLocationName())); @@ -108,18 +115,25 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) throws ValidatorException { - List locations = ProcessorData.getNewObjects(locationByName); - - List createdLocations = new ArrayList<>(); + List locations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + List createdLocations = new ArrayList<>(); try { - createdLocations.addAll(brAPILocationDAO.createBrAPILocations(locations, program.getId(), upload)); - } catch (ApiException e) { - throw new InternalServerException(e.toString(), e); + createdLocations.addAll(locationService.create(actingUser, program.getId(), locations)); + } catch (Exception e) { + log.error("Error saving location import", e); + throw new InternalServerException(e.getMessage(), e); } // Update our records createdLocations.forEach(location -> { - PendingImportObject preview = locationByName.get(location.getLocationName()); + PendingImportObject preview = locationByName.get(location.getName()); preview.setBrAPIObject(location); }); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index d05bf171f..8576b01fa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -33,6 +33,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -158,9 +159,9 @@ private void updateDependencyValues(Map mappedBrAPIImpor .forEach(this::updateTrialDbId); } - private void updateLocationDbId(BrAPILocation location) { + private void updateLocationDbId(ProgramLocation location) { this.studyByName.values().stream() - .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getLocationName())) + .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getName())) .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); } diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index d371c8103..605eabc60 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -27,17 +27,19 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.core.LocationQueryParams; import org.brapi.client.v2.modules.core.LocationsApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrApiGeoJSON; import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.request.BrAPILocationSearchRequest; import org.brapi.v2.model.core.response.BrAPILocationListResponse; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.PlaceDao; import org.breedinginsight.model.*; -import org.breedinginsight.services.brapi.BrAPIProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; import org.jooq.Configuration; import org.jooq.DSLContext; import org.jooq.Record; @@ -45,61 +47,65 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; +import java.util.stream.Collectors; import static org.breedinginsight.dao.db.Tables.*; @Singleton @Slf4j public class ProgramLocationDAO extends PlaceDao { - private DSLContext dsl; - private BrAPIProvider brAPIProvider; - private Gson gson; + private final DSLContext dsl; + private final Gson gson; - @Property(name = "brapi.server.reference-source") - protected String referenceSource; + protected final String referenceSource; + private final BrAPIDAOUtil brAPIDAOUtil; + + private final ProgramDAO programDAO; @Inject - public ProgramLocationDAO(Configuration config, DSLContext dsl, - BrAPIProvider brAPIProvider) { + public ProgramLocationDAO(Configuration config, DSLContext dsl, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO) { super(config); this.dsl = dsl; - this.brAPIProvider = brAPIProvider; this.gson = new GsonBuilder() .registerTypeAdapterFactory(new GeometryAdapterFactory()) .create(); + this.referenceSource = referenceSource; + this.brAPIDAOUtil = brAPIDAOUtil; + this.programDAO = programDAO; } // get all active locations by program id - public List getByProgramId(UUID programId) { + public List getByProgramId(UUID programId) throws ApiException { List records = getProgramLocationsQuery() .where(PLACE.PROGRAM_ID.eq(programId).and(PLACE.ACTIVE.eq(true))) .fetch(); - return parseRecords(records); + return parseRecords(records, true); } // get specified program location regardless of active status // path programId must match programId in location - public Optional getById(UUID programId, UUID locationId) { - - List records = getProgramLocationsQuery() - .where(PLACE.ID.eq(locationId).and(PLACE.PROGRAM_ID.eq(programId))) - .fetch(); - - List locations = parseRecords(records); + public Optional getById(UUID programId, UUID locationId, boolean full) throws ApiException { + List locations = getByIds(programId, List.of(locationId), full); if (locations.size() > 0){ return Optional.of(locations.get(0)); } else { return Optional.empty(); } + } + + + + public List getByIds(UUID programId, Collection locationIds, boolean full) throws ApiException { + List records = getProgramLocationsQuery() + .where(PLACE.ID.in(locationIds).and(PLACE.PROGRAM_ID.eq(programId))) + .fetch(); + return parseRecords(records, full); } private SelectOnConditionStep getProgramLocationsQuery(){ @@ -117,9 +123,10 @@ private SelectOnConditionStep getProgramLocationsQuery(){ .leftJoin(updatedByUser).on(PLACE.UPDATED_BY.eq(updatedByUser.ID)); } - private List parseRecords(List records) { + private List parseRecords(List records, boolean fetchBrAPIObject) throws ApiException { - List resultLocations = new ArrayList<>(); + Map resultLocations = new HashMap<>(); + List locationIds = new ArrayList<>(); BiUserTable createdByUser = BI_USER.as("createdByUser"); BiUserTable updatedByUser = BI_USER.as("updatedByUser"); @@ -132,17 +139,44 @@ private List parseRecords(List records) { location.setTopography(Topography.parseSQLRecord(record)); location.setCreatedByUser(org.breedinginsight.model.User.parseSQLRecord(record, createdByUser)); location.setUpdatedByUser(User.parseSQLRecord(record, updatedByUser)); - resultLocations.add(location); + resultLocations.put(location.getId(), location); + locationIds.add(location.getId()); } - return resultLocations; + if(fetchBrAPIObject) { + List brAPILocations = getBrapiLocations(locationIds, resultLocations.values().stream().findFirst().get().getProgramId()); + + if (brAPILocations.size() != resultLocations.size()) { + throw new IllegalStateException("Did not find BrAPI Location objects for each location"); + } else { + brAPILocations.forEach(brapiLocation -> { + Optional externalReference = Utilities.getExternalReference(brapiLocation.getExternalReferences(), referenceSource); + if (externalReference.isPresent()) { + ProgramLocation location = resultLocations.get(UUID.fromString(externalReference.get() + .getReferenceID())); + if (location != null) { + location.setLocationDbId(brapiLocation.getLocationDbId()); + } else { + throw new IllegalStateException("Did not find BrAPI Location for location: " + location.getId()); + } + } else { + throw new IllegalStateException("No externalReference for brapilocation: " + brapiLocation.getLocationDbId()); + } + }); + } + } + + return new ArrayList<>(resultLocations.values()); } public void createProgramLocationBrAPI(ProgramLocation location) { - BrAPIExternalReference externalReference = new BrAPIExternalReference() + BrAPIExternalReference locationIdRef = new BrAPIExternalReference() .referenceID(location.getId().toString()) .referenceSource(referenceSource); + BrAPIExternalReference programIdRef = new BrAPIExternalReference() + .referenceID(location.getProgramId().toString()) + .referenceSource(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName())); BrAPILocation brApiLocation = new BrAPILocation() .abbreviation(location.getAbbreviation()) @@ -155,7 +189,7 @@ public void createProgramLocationBrAPI(ProgramLocation location) { .documentationURL(location.getDocumentationUrl()) .environmentType(location.getEnvironmentType() != null ? location.getEnvironmentType().getName() : null) .exposure(location.getExposure()) - .externalReferences(List.of(externalReference)) + .externalReferences(List.of(locationIdRef, programIdRef)) //.instituteAddress() do not keep this in our model //.instituteName() do not keep this in our model .locationName(location.getName()) @@ -167,9 +201,10 @@ public void createProgramLocationBrAPI(ProgramLocation location) { // POST locations to each brapi service // TODO: If there is a failure after the first brapi service, roll back all before the failure. try { - List locationsAPIs = brAPIProvider.getAllUniqueLocationsAPI(); - for (LocationsApi locationsAPI: locationsAPIs){ - locationsAPI.locationsPost(List.of(brApiLocation)); + LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(location.getProgramId())); + ApiResponse brapiResponse = locationsAPI.locationsPost(List.of(brApiLocation)); + if(brapiResponse.getBody().getResult().getData().size() == 1) { + location.setLocationDbId(brapiResponse.getBody().getResult().getData().get(0).getLocationDbId()); } } catch (ApiException e) { log.warn(Utilities.generateApiExceptionLogMessage(e)); @@ -180,29 +215,14 @@ public void createProgramLocationBrAPI(ProgramLocation location) { public void updateProgramLocationBrAPI(ProgramLocation location) { - LocationQueryParams searchRequest = new LocationQueryParams() - .externalReferenceID(location.getId().toString()) - .externalReferenceSource(referenceSource); - - // Location goes in all of the clients - // TODO: If there is a failure after the first brapi service, roll back all before the failure. - List locationsAPIs = brAPIProvider.getAllUniqueLocationsAPI(); - for (LocationsApi locationsAPI: locationsAPIs){ - - // Get existing brapi location - ApiResponse brApiLocations; - try { - brApiLocations = locationsAPI.locationsGet(searchRequest); - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - throw new HttpServerException("Could not find location in BrAPI service."); - } + try { + List brApiLocations = getBrapiLocations(List.of(location.getId()), location.getProgramId()); - if (brApiLocations.getBody().getResult().getData().size() != 1){ + if (brApiLocations.size() != 1){ throw new HttpServerException("Could not find unique location in BrAPI service."); } - BrAPILocation brApiLocation = brApiLocations.getBody().getResult().getData().get(0); + BrAPILocation brApiLocation = brApiLocations.get(0); //TODO: Need to add archived/not archived when available in brapi brApiLocation.setAbbreviation(location.getAbbreviation()); @@ -223,15 +243,27 @@ public void updateProgramLocationBrAPI(ProgramLocation location) { brApiLocation.setSlope(location.getSlope() != null ? location.getSlope().toPlainString() : null); brApiLocation.setTopography(location.getTopography() != null ? location.getTopography().getName() : null); - try { - locationsAPI.locationsLocationDbIdPut(brApiLocation.getLocationDbId(), brApiLocation); - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - throw new HttpServerException("Could not find location in BrAPI service."); - } + LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(location.getProgramId())); + locationsAPI.locationsLocationDbIdPut(brApiLocation.getLocationDbId(), brApiLocation); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new HttpServerException("Could not find location in BrAPI service."); } } + private List getBrapiLocations(List locationIds, UUID programId) throws ApiException { + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() + .externalReferenceIDs(locationIds.stream().map(UUID::toString).collect(Collectors.toList())) + .externalReferenceSources(List.of(referenceSource)); + + // Location goes in all of the clients + // TODO: If there is a failure, roll back all before the failure. + LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(programId)); + + // Get existing brapi location + return brAPIDAOUtil.search(locationsAPI::searchLocationsPost, locationsAPI::searchLocationsSearchResultsDbIdGet, searchRequest); + } + private BrApiGeoJSON getClientGeoJson(ProgramLocation location) { BrApiGeoJSON geoJson = null; if (location.getCoordinates() != null) { @@ -245,4 +277,52 @@ private BrApiGeoJSON getClientGeoJson(ProgramLocation location) { } + public List getByDbIds(Collection locationDbIds, UUID programId) throws ApiException { + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() + .locationDbIds(new ArrayList<>(locationDbIds)) + .externalReferenceIDs(List.of(programId.toString())) + .externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName()))); + + return getProgramLocationsByBrAPISearch(programId, searchRequest); + } + + public List getByNames(List names, UUID programId) throws ApiException { + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() + .locationNames(new ArrayList<>(names)) + .externalReferenceIDs(List.of(programId.toString())) + .externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName()))); + + return getProgramLocationsByBrAPISearch(programId, searchRequest); + } + + @NotNull + private List getProgramLocationsByBrAPISearch(UUID programId, BrAPILocationSearchRequest searchRequest) throws ApiException { + LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(programId)); + List searchResult = brAPIDAOUtil.search(locationsAPI::searchLocationsPost, locationsAPI::searchLocationsSearchResultsDbIdGet, searchRequest); + + Map brapiLocationById = new HashMap<>(); + searchResult.forEach(brAPILocation -> { + Optional xref = Utilities.getExternalReference(brAPILocation.getExternalReferences(), referenceSource); + if(xref.isPresent()) { + brapiLocationById.put(UUID.fromString(xref.get().getReferenceID()), brAPILocation); + } else { + throw new IllegalStateException(String.format("Location (by dbid): %s does not have any external references", brAPILocation.getLocationDbId())); + } + }); + + List records = getProgramLocationsQuery() + .where(PLACE.ID.in(brapiLocationById.keySet()).and(PLACE.PROGRAM_ID.eq(programId))) + .fetch(); + List programLocations = parseRecords(records, false); + if(programLocations.size() != brapiLocationById.size()) { + throw new IllegalStateException("Didn't find all locations by id"); + } else { + programLocations.forEach(location -> { + BrAPILocation brAPILocation = brapiLocationById.get(location.getId()); + location.setLocationDbId(brAPILocation.getLocationDbId()); + }); + } + + return programLocations; + } } diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index 4b10d257e..ff1d8be91 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -29,6 +29,7 @@ import org.brapi.client.v2.BrAPIClient; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.phenotype.VariableQueryParams; +import org.brapi.client.v2.modules.core.LocationsApi; import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.*; @@ -442,10 +443,8 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra // TODO: If there is a failure after the first brapi service, roll back all before the failure. ApiResponse createdVariables = null; try { - List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); - for (ObservationVariablesApi variablesAPI: variablesAPIS){ - createdVariables = variablesAPI.variablesPost(brApiVariables); - } + ObservationVariablesApi variablesAPI = new ObservationVariablesApi(programDAO.getCoreClient(program.getId())); + createdVariables = variablesAPI.variablesPost(brApiVariables); } catch (ApiException e) { log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); diff --git a/src/main/java/org/breedinginsight/model/BrAPIConstants.java b/src/main/java/org/breedinginsight/model/BrAPIConstants.java index ab767ff39..dde083154 100644 --- a/src/main/java/org/breedinginsight/model/BrAPIConstants.java +++ b/src/main/java/org/breedinginsight/model/BrAPIConstants.java @@ -4,7 +4,7 @@ public enum BrAPIConstants { SYSTEM_DEFAULT("System Default"), - REPLICATE( "replicate"), + REPLICATE( "rep"), BLOCK( "block"); private String value; diff --git a/src/main/java/org/breedinginsight/model/ProgramLocation.java b/src/main/java/org/breedinginsight/model/ProgramLocation.java index 05ff52d4d..0067b5d11 100644 --- a/src/main/java/org/breedinginsight/model/ProgramLocation.java +++ b/src/main/java/org/breedinginsight/model/ProgramLocation.java @@ -45,6 +45,7 @@ public class ProgramLocation extends PlaceEntity { private Accessibility accessibility; private EnvironmentType environmentType; private Topography topography; + private String locationDbId; private User createdByUser; private User updatedByUser; diff --git a/src/main/java/org/breedinginsight/services/ProgramLocationService.java b/src/main/java/org/breedinginsight/services/ProgramLocationService.java index da78aa6e2..7aa1c504d 100644 --- a/src/main/java/org/breedinginsight/services/ProgramLocationService.java +++ b/src/main/java/org/breedinginsight/services/ProgramLocationService.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.dao.db.tables.pojos.PlaceEntity; @@ -35,9 +36,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.time.OffsetDateTime; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; @Slf4j @Singleton @@ -68,7 +67,7 @@ public ProgramLocationService(ProgramLocationDAO programLocationDao, this.dsl = dsl; } - public List getByProgramId(UUID programId) throws DoesNotExistException { + public List getByProgramId(UUID programId) throws DoesNotExistException, ApiException { if (!programService.exists(programId)) { throw new DoesNotExistException("Program id does not exist"); @@ -77,9 +76,8 @@ public List getByProgramId(UUID programId) throws DoesNotExistE return programLocationDao.getByProgramId(programId); } - public Optional getById(UUID programId, UUID locationId) { - - return programLocationDao.getById(programId, locationId); + public Optional getById(UUID programId, UUID locationId) throws ApiException { + return programLocationDao.getById(programId, locationId, false); } private UUID validateCountryId(ProgramLocationRequest programLocationRequest) throws UnprocessableEntityException { @@ -195,6 +193,20 @@ boolean isListPointsLatLngValid(List points) { return true; } + public List create(AuthenticatedUser actingUser, UUID programId, List newLocations) throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + // check if programId exists + if (!programService.exists(programId)) { + throw new DoesNotExistException("Program id does not exist"); + } + + List ret = new ArrayList<>(); + for(ProgramLocationRequest newLoc : newLocations) { + ret.add(createLocation(actingUser, programId, newLoc)); + } + + return ret; + } + public ProgramLocation create(AuthenticatedUser actingUser, UUID programId, ProgramLocationRequest programLocationRequest) @@ -205,6 +217,10 @@ public ProgramLocation create(AuthenticatedUser actingUser, throw new DoesNotExistException("Program id does not exist"); } + return createLocation(actingUser, programId, programLocationRequest); + } + + private ProgramLocation createLocation(AuthenticatedUser actingUser, UUID programId, ProgramLocationRequest programLocationRequest) throws UnprocessableEntityException, MissingRequiredInfoException { // validate fields UUID countryId = validateCountryId(programLocationRequest); UUID environmentTypeId = validateEnvironmentTypeId(programLocationRequest); @@ -214,22 +230,22 @@ public ProgramLocation create(AuthenticatedUser actingUser, // parse and create the program location object PlaceEntity placeEntity = PlaceEntity.builder() - .programId(programId) - .name(programLocationRequest.getName()) - .countryId(countryId) - .environmentTypeId(environmentTypeId) - .accessibilityId(accessibilityId) - .topographyId(topographyId) - .abbreviation(programLocationRequest.getAbbreviation()) - .coordinates(coordinates != null ? JSONB.valueOf(coordinates) : null) - .coordinateUncertainty(programLocationRequest.getCoordinateUncertainty()) - .coordinateDescription(programLocationRequest.getCoordinateDescription()) - .slope(programLocationRequest.getSlope()) - .exposure(programLocationRequest.getExposure()) - .documentationUrl(programLocationRequest.getDocumentationUrl()) - .createdBy(actingUser.getId()) - .updatedBy(actingUser.getId()) - .build(); + .programId(programId) + .name(programLocationRequest.getName()) + .countryId(countryId) + .environmentTypeId(environmentTypeId) + .accessibilityId(accessibilityId) + .topographyId(topographyId) + .abbreviation(programLocationRequest.getAbbreviation()) + .coordinates(coordinates != null ? JSONB.valueOf(coordinates) : null) + .coordinateUncertainty(programLocationRequest.getCoordinateUncertainty()) + .coordinateDescription(programLocationRequest.getCoordinateDescription()) + .slope(programLocationRequest.getSlope()) + .exposure(programLocationRequest.getExposure()) + .documentationUrl(programLocationRequest.getDocumentationUrl()) + .createdBy(actingUser.getId()) + .updatedBy(actingUser.getId()) + .build(); ProgramLocation location = null; @@ -237,7 +253,7 @@ public ProgramLocation create(AuthenticatedUser actingUser, // This is warped in a transaction so if the BrAPI save call fails, the BI database insert is rolled back. location = dsl.transactionResult(configuration -> { programLocationDao.insert(placeEntity); - ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId()).get(); + ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).get(); // Add location to brapi service programLocationDao.createProgramLocationBrAPI(progLocation); @@ -285,7 +301,7 @@ public ProgramLocation update(AuthenticatedUser actingUser, // This is warped in a transaction so if the BrAPI update post fails, the BI database update is rolled back. ProgramLocation location = dsl.transactionResult(configuration -> { programLocationDao.update(placeEntity); - ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId()).get(); + ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).get(); // Update location in brapi service programLocationDao.updateProgramLocationBrAPI(progLocation); @@ -321,4 +337,12 @@ public void delete(UUID locationId) throws DoesNotExistException { programLocationDao.delete(placeEntity); } + + public List getLocationsByDbId(Collection locationDbIds, UUID programId) throws ApiException { + return programLocationDao.getByDbIds(locationDbIds, programId); + } + + public List getLocationsByName(List names, UUID programId) throws ApiException { + return programLocationDao.getByNames(names, programId); + } } diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 6a4e52238..d7b35801b 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -95,6 +95,21 @@ public Optional getById(UUID programId) { return Optional.of(program); } + public Optional getByKey(String programKey) { + + List programs = dao.getProgramByKey(programKey); + + if (programs.size() <= 0) { + return Optional.empty(); + } + + Program program = programs.get(0); + BrAPIProgram brapiProgram = dao.getProgramBrAPI(program); + program.setBrAPIProperties(brapiProgram); + + return Optional.of(program); + } + public List getAll(AuthenticatedUser actingUser){ /* Get all of the programs the user has access to */ List enrolledProgramIds = securityService.getEnrolledProgramIds(actingUser); diff --git a/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java b/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java index d0a194bc2..f07c1a3df 100644 --- a/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java +++ b/src/main/java/org/breedinginsight/services/exceptions/ValidatorException.java @@ -18,9 +18,11 @@ package org.breedinginsight.services.exceptions; import lombok.Getter; +import lombok.ToString; import org.breedinginsight.api.model.v1.response.ValidationErrors; @Getter +@ToString public class ValidatorException extends Exception { private ValidationErrors errors; diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index b8001ac2e..a74235c4d 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -31,6 +31,7 @@ import org.brapi.v2.model.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import javax.inject.Inject; import javax.inject.Singleton; import java.time.Duration; import java.util.ArrayList; @@ -43,14 +44,21 @@ @Slf4j public class BrAPIDAOUtil { - @Property(name = "brapi.search.wait-time") - private int searchWaitTime; - @Property(name = "brapi.read-timeout") - private Duration searchTimeout; - @Property(name = "brapi.page-size") - private int pageSize; - @Property(name = "brapi.post-group-size") - private int postGroupSize; + private final int searchWaitTime; + private final Duration searchTimeout; + private final int pageSize; + private final int postGroupSize; + + @Inject + public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, + @Property(name = "brapi.read-timeout") Duration searchTimeout, + @Property(name = "brapi.page-size") int pageSize, + @Property(name = "brapi.post-group-size") int postGroupSize) { + this.searchWaitTime = searchWaitTime; + this.searchTimeout = searchTimeout; + this.pageSize = pageSize; + this.postGroupSize = postGroupSize; + } public List search(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, @@ -280,7 +288,9 @@ public List post(List brapiObjects, } List data = result.getData(); // TODO: Maybe move this outside of the loop - if (data.size() != postChunk.size()) throw new ApiException("Number of brapi objects returned does not equal number sent"); + if (data.size() != postChunk.size()) { + throw new ApiException("Number of brapi objects returned does not equal number sent"); + } listResult.addAll(data); finished += data.size(); currentRightBorder += postGroupSize; diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 139fbf37e..8dbde7fad 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -60,7 +60,7 @@ public static boolean containsCaseInsensitive(String target, List list){ * @return the formatted string */ public static String appendProgramKey(String original, String programKey, String additionalKeyData) { - if(StringUtils.isNotEmpty(additionalKeyData)) { + if(StringUtils.isNotBlank(additionalKeyData)) { return String.format("%s [%s-%s]", original, programKey, additionalKeyData); } else { return String.format("%s [%s]", original, programKey); @@ -80,7 +80,7 @@ public static String appendProgramKey( String original, String programKey ){ * @return */ public static String removeProgramKey(String original, String programKey, String additionalKeyData) { - if(StringUtils.isNotEmpty(additionalKeyData)) { + if(StringUtils.isNotBlank(additionalKeyData)) { String keyValue = String.format(" [%s-%s]", programKey, additionalKeyData); return original.replace(keyValue, ""); } else { diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java new file mode 100644 index 000000000..af56ea1e8 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -0,0 +1,755 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.BrAPIPositionCoordinateTypeEnum; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.SpeciesService; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.junit.platform.commons.util.StringUtils; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ExperimentFileImportTest extends BrAPITest { + + private FannyPack securityFp; + private String mappingId; + private BiUserEntity testUser; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Property(name = "brapi.server.core-url") + private String BRAPI_URL; + + @Inject + private SpeciesService speciesService; + @Inject + private UserDAO userDAO; + @Inject + private DSLContext dsl; + + @Inject + private SpeciesDAO speciesDAO; + + @Inject + private ProgramDAO programDAO; + + @Inject + private ProgramService programService; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private ImportTestUtils importTestUtils; + + @Inject + private OntologyService ontologyService; + + @Inject + private BrAPITrialDAO brAPITrialDAO; + + @Inject + private BrAPIStudyDAO brAPIStudyDAO; + + @Inject + private BrAPIObservationUnitDAO ouDAO; + + @Inject + private ProgramLocationService locationService; + + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + private BrAPIObservationDAO observationDAO; + + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + public void setup() { + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "ExperimentsTemplateMap"); + mappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); + securityFp = (FannyPack) setupObjects.get("securityFp"); + + } + + /* + Tests + - new experiment + - existing experiment, new env + - new env, existing location + - new experiment, missing required cols (check all required cols) + - new env, missing required cols (check all req cols) + - new exp/env with observations + - new exp/env with invalid observations + - existing env with missing OU ID + - existing env with new observations + - existing env that already has obs + */ + + @Test + @SneakyThrows + public void importNewExpNewLocNoObsSuccess() { + Program program = createProgram("New Exp and Loc", "NEXPL", "NEXPL", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map validRow = new HashMap<>(); + validRow.put(Columns.GERMPLASM_GID, "1"); + validRow.put(Columns.TEST_CHECK, "T"); + validRow.put(Columns.EXP_TITLE, "Test Exp"); + validRow.put(Columns.EXP_DESCRIPTION, "Test Description"); + validRow.put(Columns.EXP_UNIT, "Plot"); + validRow.put(Columns.EXP_TYPE, "Phenotyping"); + validRow.put(Columns.ENV, "Test Env"); + validRow.put(Columns.ENV_LOCATION, "Location A"); + validRow.put(Columns.ENV_YEAR, "2023"); + validRow.put(Columns.EXP_UNIT_ID, "a-1"); + validRow.put(Columns.REP_NUM, "1"); + validRow.put(Columns.BLOCK_NUM, "1"); + validRow.put(Columns.ROW, "1"); + validRow.put(Columns.COLUMN, "1"); + validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(validRow, row, program, null); + } + + @Test + @SneakyThrows + public void importNewEnvExistingExpNoObsSuccess() { + Program program = createProgram("New Env Existing Exp", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + Map newEnv = new HashMap<>(); + newEnv.put(Columns.GERMPLASM_GID, "1"); + newEnv.put(Columns.TEST_CHECK, "T"); + newEnv.put(Columns.EXP_TITLE, "Test Exp"); + newEnv.put(Columns.EXP_UNIT, "Plot"); + newEnv.put(Columns.EXP_TYPE, "Phenotyping"); + newEnv.put(Columns.ENV, "New Trial Existing Exp"); + newEnv.put(Columns.ENV_LOCATION, "Location A"); + newEnv.put(Columns.ENV_YEAR, "2023"); + newEnv.put(Columns.EXP_UNIT_ID, "a-1"); + newEnv.put(Columns.REP_NUM, "1"); + newEnv.put(Columns.BLOCK_NUM, "1"); + newEnv.put(Columns.ROW, "1"); + newEnv.put(Columns.COLUMN, "1"); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newEnv, row, program, null); + } + + @Test + @SneakyThrows + public void verifyMissingDataThrowsError() { + Program program = createProgram("Missing Req Cols", "MISS", "MISS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + + Map base = new HashMap<>(); + base.put(Columns.GERMPLASM_GID, "1"); + base.put(Columns.TEST_CHECK, "T"); + base.put(Columns.EXP_TITLE, "Missing Req Cols"); + base.put(Columns.EXP_UNIT, "Plot"); + base.put(Columns.EXP_TYPE, "Phenotyping"); + base.put(Columns.ENV, "Missing GID"); + base.put(Columns.ENV_LOCATION, "Location A"); + base.put(Columns.ENV_YEAR, "2023"); + base.put(Columns.EXP_UNIT_ID, "a-1"); + base.put(Columns.REP_NUM, "1"); + base.put(Columns.BLOCK_NUM, "1"); + + Map noGID = new HashMap<>(base); + noGID.remove(Columns.GERMPLASM_GID); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID); + + Map noExpTitle = new HashMap<>(base); + noExpTitle.remove(Columns.EXP_TITLE); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE); + + Map noExpUnit = new HashMap<>(base); + noExpUnit.remove(Columns.EXP_UNIT); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT); + + Map noExpType = new HashMap<>(base); + noExpType.remove(Columns.EXP_TYPE); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpType), null), Columns.EXP_TYPE); + + Map noEnv = new HashMap<>(base); + noEnv.remove(Columns.ENV); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnv), null), Columns.ENV); + + Map noEnvLoc = new HashMap<>(base); + noEnvLoc.remove(Columns.ENV_LOCATION); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION); + + Map noExpUnitId = new HashMap<>(base); + noExpUnitId.remove(Columns.EXP_UNIT_ID); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID); + + Map noExpRep = new HashMap<>(base); + noExpRep.remove(Columns.REP_NUM); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpRep), null), Columns.REP_NUM); + + Map noExpBlock = new HashMap<>(base); + noExpBlock.remove(Columns.BLOCK_NUM); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM); + + Map noEnvYear = new HashMap<>(base); + noEnvYear.remove(Columns.ENV_YEAR); + uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR); + } + + @Test + @SneakyThrows + public void importNewExpWithObs() { + List traits = createTraits(1); + Program program = createProgram("New Exp with Observations", "EXPOBS", "EXPOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + System.out.println(row); + + assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExp, row, program, traits); + } + + @Test + @SneakyThrows + public void verifyFailureImportNewExpWithInvalidObs() { + List traits = createTraits(1); + Program program = createProgram("Invalid Observations", "INVOBS", "INVOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "Red"); + + uploadAndVerifyFailure(program, writeDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName()); + } + + @Test + @SneakyThrows + public void verifyFailureNewOuExistingEnv() { + Program program = createProgram("New OU Exising Env", "FAILOU", "FAILOU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + Map newOU = new HashMap<>(newExp); + newOU.put(Columns.EXP_UNIT_ID, "a-2"); + newOU.put(Columns.ROW, "1"); + newOU.put(Columns.COLUMN, "2"); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(List.of(newOU), null), null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Units are missing Observation Unit Id.")); + } + + @Test + @SneakyThrows + public void importNewObsExisingOu() { + List traits = createTraits(1); + Program program = createProgram("New Obs Existing OU", "OUOBS", "OUOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObservation = new HashMap<>(); + newObservation.put(Columns.GERMPLASM_GID, "1"); + newObservation.put(Columns.TEST_CHECK, "T"); + newObservation.put(Columns.EXP_TITLE, "Test Exp"); + newObservation.put(Columns.EXP_UNIT, "Plot"); + newObservation.put(Columns.EXP_TYPE, "Phenotyping"); + newObservation.put(Columns.ENV, "New Env"); + newObservation.put(Columns.ENV_LOCATION, "Location A"); + newObservation.put(Columns.ENV_YEAR, "2023"); + newObservation.put(Columns.EXP_UNIT_ID, "a-1"); + newObservation.put(Columns.REP_NUM, "1"); + newObservation.put(Columns.BLOCK_NUM, "1"); + newObservation.put(Columns.ROW, "1"); + newObservation.put(Columns.COLUMN, "1"); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newObservation), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + System.out.println("row:" + row); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newObservation, row, program, traits); + } + + @Test + @SneakyThrows + public void importNewObsExisingOuWithExistingObs() { + fail(); + } + + private Map assertRowSaved(Map expected, JsonObject actual, Program program, List traits) throws ApiException { + Map ret = new HashMap<>(); + + List trials = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)expected.get(Columns.EXP_TITLE), program.getKey())), program); + assertFalse(trials.isEmpty()); + BrAPITrial trial = trials.get(0); + Optional trialIdXref = Utilities.getExternalReference(trial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + + List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program); + assertFalse(studies.isEmpty()); + BrAPIStudy study = studies.get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + List locations = locationService.getLocationsByDbId(List.of(study.getLocationDbId()), program.getId()); + assertFalse(locations.isEmpty()); + ProgramLocation location = locations.get(0); + + List germplasms = germplasmDAO.getGermplasmsByDBID(List.of(ou.getGermplasmDbId()), program.getId()); + assertFalse(germplasms.isEmpty()); + BrAPIGermplasm germplasm = germplasms.get(0); + +// assertNotNull(actual.get("trial")); +// BrAPITrial trial = gson.fromJson(actual.getAsJsonObject("trial").getAsJsonObject("brAPIObject"), BrAPITrial.class); + ret.put("trial", trial); + +// assertNotNull(actual.get("study")); +// BrAPIStudy study = gson.fromJson(actual.getAsJsonObject("study").getAsJsonObject("brAPIObject"), BrAPIStudy.class); + ret.put("study", study); + +// assertNotNull(actual.get("location")); +// ProgramLocation location = gson.fromJson(actual.getAsJsonObject("location").getAsJsonObject("brAPIObject"), ProgramLocation.class); + ret.put("location", location); + +// assertNotNull(actual.get("observationUnit")); +// BrAPIObservationUnit ou = gson.fromJson(actual.getAsJsonObject("observationUnit").getAsJsonObject("brAPIObject"), BrAPIObservationUnit.class); + ret.put("observationUnit", ou); + +// assertNotNull(actual.get("germplasm")); +// BrAPIGermplasm germplasm = gson.fromJson(actual.getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"), BrAPIGermplasm.class); + ret.put("germplasm", germplasm); + + List observations = null; + if(traits != null) { + observations = observationDAO.getObservationsByStudyName(List.of(study.getStudyName()), program); + assertFalse(observations.isEmpty()); + +// assertNotNull(actual.get("observations")); +// observations = gson.fromJson(actual.get("observations"), new TypeToken>(){}.getType()); + ret.put("observations", observations); + } + + assertNotNull(germplasm.getGermplasmName()); + assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); + if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { + assertEquals(expected.get(Columns.TEST_CHECK), + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); + } + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); + assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); + assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_LOCATION), study.getLocationName()); + assertEquals(expected.get(Columns.ENV_LOCATION), location.getName()); + //TODO figure out how to get the actual season value +// assertEquals(expected.getInt(Columns.ENV_YEAR), Integer.parseInt(study.getSeasons().get(0))); + assertEquals(expected.get(Columns.EXP_UNIT_ID), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + BrAPIObservationUnitLevelRelationship rep = null; + BrAPIObservationUnitLevelRelationship block = null; + for (BrAPIObservationUnitLevelRelationship rel : ou.getObservationUnitPosition().getObservationLevelRelationships()) { + if ("rep".equals(rel.getLevelName()) && rep == null) { + rep = rel; + } else if ("block".equals(rel.getLevelName()) && block == null) { + block = rel; + } + } + assertNotNull(rep); + assertNotNull(block); + assertEquals(expected.get(Columns.REP_NUM), rep.getLevelCode()); + assertEquals(expected.get(Columns.BLOCK_NUM), block.getLevelCode()); + if(expected.containsKey(Columns.ROW)) { + assertEquals(expected.get(Columns.ROW), ou.getObservationUnitPosition().getPositionCoordinateX()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_ROW, ou.getObservationUnitPosition().getPositionCoordinateXType()); + } + if(expected.containsKey(Columns.COLUMN)) { + assertEquals(expected.get(Columns.COLUMN), ou.getObservationUnitPosition().getPositionCoordinateY()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_COL, ou.getObservationUnitPosition().getPositionCoordinateYType()); + } + if(expected.containsKey(Columns.TREATMENT_FACTORS) && StringUtils.isNotBlank((String)expected.get(Columns.TREATMENT_FACTORS))) { + assertEquals(expected.get(Columns.TREATMENT_FACTORS), ou.getTreatments().get(0).getFactor()); + } + + if(traits != null) { + List expectedVariableObservation = new ArrayList<>(); + List actualVariableObservation = new ArrayList<>(); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", observation.getObservationVariableName(), observation.getValue()))); + for(Trait trait : traits) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); + } + + return ret; + } + + private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + ProgramRequest programRequest1 = ProgramRequest.builder() + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); + + + TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); + + // Get main program + Program program = programService.getByKey(key).get(); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + + if(germplasm != null && !germplasm.isEmpty()) { + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", referenceSource)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.importBrAPIGermplasm(germplasm, program.getId(), null); + } + + if(traits != null && !traits.isEmpty()) { + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + } + + return program; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private JsonObject uploadAndVerifyFailure(Program program, File file, String expectedColumnError) throws InterruptedException, IOException { + Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.TEST_CHECK).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_TITLE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_DESCRIPTION).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_UNIT).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.EXP_TYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ENV).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ENV_LOCATION).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ENV_YEAR).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.EXP_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.REP_NUM).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.BLOCK_NUM).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.TREATMENT_FACTORS).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + + if(traits != null) { + traits.forEach(trait -> { + columns.add(Column.builder().value(trait.getObservationVariableName()).dataType(Column.ColumnDataType.STRING).build()); + }); + } + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + +} diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java similarity index 80% rename from src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java rename to src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index ade545d08..b52dbde0a 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer; import com.google.gson.*; -import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -31,7 +30,6 @@ import org.breedinginsight.daos.BreedingMethodDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; -import org.breedinginsight.model.Species; import org.breedinginsight.services.SpeciesService; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; @@ -48,17 +46,17 @@ import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; -import static io.micronaut.http.HttpRequest.PUT; import static org.junit.jupiter.api.Assertions.*; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class GermplasmTemplateMap extends BrAPITest { +public class GermplasmFileImportTest extends BrAPITest { - private FannyPack fp; + private static final String GERM_LIST_NAME = "germplasmListName"; + private static final String GERM_LIST_DESC = "germplasmListDescription"; private Program validProgram; - private String germplasmImportId; + private String germplasmMappingId; private BiUserEntity testUser; @Property(name = "brapi.server.reference-source") @@ -75,65 +73,23 @@ public class GermplasmTemplateMap extends BrAPITest { @Inject private DSLContext dsl; + private ImportTestUtils importTestUtils; + @Inject @Client("/${micronaut.bi.api.version}") RxHttpClient client; - private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) - .create(); + .create(); @BeforeAll public void setup() { - fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); - var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); - var brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); - - // Set up BrAPI - super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); - - // Species - Species validSpecies = speciesService.getAll().get(0); - SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); - - // Insert program - ProgramRequest program = ProgramRequest.builder() - .name("Test Program") - .species(speciesRequest) - .key("TEST") - .build(); - validProgram = insertAndFetchTestProgram(program); - - // Get germplasm system import - Flowable> call = client.exchange( - GET("/import/mappings?importName=germplasmtemplatemap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - germplasmImportId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); - - testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); - dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); - dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); - } - - public Program insertAndFetchTestProgram(ProgramRequest programRequest) { - - Flowable> call = client.exchange( - POST("/programs/", gson.toJson(programRequest)) - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpResponse response = call.blockingFirst(); - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - Program program = gson.fromJson(result, Program.class); - return program; + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "GermplasmTemplateMap"); + validProgram = (Program) setupObjects.get("program"); + germplasmMappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); } // Tests @@ -164,13 +120,12 @@ public Program insertAndFetchTestProgram(ProgramRequest programRequest) { public void minimalImportUserSpecifiedEntryNumbersSuccess() { File file = new File("src/test/resources/files/germplasm_import/minimal_germplasm_import.csv"); String listName = "MinimalList"; - String listDescription = null; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -184,7 +139,7 @@ public void minimalImportUserSpecifiedEntryNumbersSuccess() { } // Check the germplasm list - checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), null, germplasmNames); } @Test @@ -196,12 +151,12 @@ public void fullImportPreviewSuccess() { String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, false); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), false, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -217,9 +172,9 @@ public void fullImportPreviewSuccess() { assertEquals(fileData.getString(i, "Name"), germplasm.get("germplasmName").getAsString()); JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); // Created Date (not present) - assertTrue(!additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is present, but should not be"); + assertFalse(additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is present, but should not be"); // Accession Number (not present) - assertTrue(!germplasm.has("accessionNumber"), "accessionNumber is present, but should not be"); + assertFalse(germplasm.has("accessionNumber"), "accessionNumber is present, but should not be"); // Pedigree (display names) String pedigree = germplasm.get("pedigree").getAsString(); String mother = !pedigree.isBlank() ? pedigree.split("/")[0] : null; @@ -235,7 +190,7 @@ public void fullImportPreviewSuccess() { JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); for (JsonElement reference: externalReferences) { String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); - assertTrue(referenceSource != BRAPI_REFERENCE_SOURCE, "Germplasm UUID was present, but should not be"); + assertNotSame(referenceSource, BRAPI_REFERENCE_SOURCE, "Germplasm UUID was present, but should not be"); } // Synonyms @@ -256,12 +211,12 @@ public void fullImportCommitSuccess() { String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -297,7 +252,7 @@ public void fullImportCommitSuccess() { // External Reference germplasm JsonArray externalReferences = germplasm.getAsJsonArray("externalReferences"); - Boolean referenceFound = false; + boolean referenceFound = false; for (JsonElement reference: externalReferences) { String referenceSource = reference.getAsJsonObject().get("referenceSource").getAsString(); if (referenceSource.equals(BRAPI_REFERENCE_SOURCE)) { @@ -330,12 +285,12 @@ public void duplicateNameMarksDuplicates() { String listName = "DupNamesList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, false); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), false, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -357,12 +312,12 @@ public void duplicateNameMarksDuplicates() { public void OnlyMaleParentPreviewSuccess() { File file = new File("src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv"); - Flowable> call = uploadDataFile(file, "NoFemaleParentList", null, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "NoFemaleParentList"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -378,7 +333,7 @@ public void OnlyMaleParentPreviewSuccess() { @SneakyThrows public void missingRequiredUserInput() { File file = new File("src/test/resources/files/germplasm_import/female_dbid_not_exist.csv"); - Flowable> call = uploadDataFile(file, null, null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(), true, client, validProgram, germplasmMappingId); HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { HttpResponse response = call.blockingFirst(); }); @@ -394,11 +349,11 @@ public void germplasmListNameDuplicateError() { String listName = "FullList"; String listDescription = "A full import"; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName, GERM_LIST_DESC, listDescription), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.listNameAlreadyExists, result.getAsJsonObject("progress").get("message").getAsString()); @@ -408,12 +363,12 @@ public void germplasmListNameDuplicateError() { @SneakyThrows public void femaleParentDbIdNotExistError() { File file = new File("src/test/resources/files/germplasm_import/female_dbid_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("1000", "1001", "1002"); @@ -425,12 +380,12 @@ public void femaleParentDbIdNotExistError() { @SneakyThrows public void femaleParentEntryNumberNotExistError() { File file = new File("src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("1", "2", "3"); @@ -442,12 +397,12 @@ public void femaleParentEntryNumberNotExistError() { @SneakyThrows public void maleParentDbIdNotExistError() { File file = new File("src/test/resources/files/germplasm_import/male_dbid_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("100", "101", "102"); @@ -459,12 +414,12 @@ public void maleParentDbIdNotExistError() { @SneakyThrows public void maleParentEntryNumberNotExistError() { File file = new File("src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List missingDbIds = List.of("1", "2", "3"); @@ -476,12 +431,12 @@ public void maleParentEntryNumberNotExistError() { @SneakyThrows public void badBreedingMethods() { File file = new File("src/test/resources/files/germplasm_import/bad_breeding_methods.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -502,12 +457,12 @@ public void badBreedingMethods() { @SneakyThrows public void someEntryNumbersError() { File file = new File("src/test/resources/files/germplasm_import/some_entry_numbers.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.missingEntryNumbersMsg, result.getAsJsonObject("progress").get("message").getAsString()); @@ -517,12 +472,12 @@ public void someEntryNumbersError() { @SneakyThrows public void duplicateEntryNumbersError() { File file = new File("src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv"); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); List dups = List.of("1", "3"); @@ -537,7 +492,7 @@ public void nonNumericEntryNumbersError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -558,7 +513,7 @@ public void emptyRequiredFieldsError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -571,11 +526,11 @@ public void emptyRequiredFieldsError() { assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); JsonArray rowErrors = JsonParser.parseString((String) e.getResponse().getBody().get()).getAsJsonObject().getAsJsonArray("rowErrors"); - assertTrue(rowErrors.size() == 2, "Wrong number of row errors returned"); + assertEquals(2, rowErrors.size(), "Wrong number of row errors returned"); JsonObject rowError1 = rowErrors.get(0).getAsJsonObject(); JsonArray errors = rowError1.getAsJsonArray("errors"); - assertTrue(errors.size() == 1, "Not enough errors were returned"); + assertEquals(1, errors.size(), "Not enough errors were returned"); JsonObject error = errors.get(0).getAsJsonObject(); assertEquals(422, error.get("httpStatusCode").getAsInt(), "Incorrect http status code"); assertEquals("Name", error.get("field").getAsString(), "Incorrect field name"); @@ -583,7 +538,7 @@ public void emptyRequiredFieldsError() { JsonObject rowError2 = rowErrors.get(1).getAsJsonObject(); JsonArray errors2 = rowError2.getAsJsonArray("errors"); - assertTrue(errors2.size() == 1, "Not enough errors were returned"); + assertEquals(1, errors2.size(), "Not enough errors were returned"); JsonObject error2 = errors2.get(0).getAsJsonObject(); assertEquals(422, error2.get("httpStatusCode").getAsInt(), "Incorrect http status code"); assertEquals("Source", error2.get("field").getAsString(), "Incorrect field name"); @@ -595,13 +550,12 @@ public void emptyRequiredFieldsError() { public void headerCaseInsensitive() { File file = new File("src/test/resources/files/germplasm_import/germplasm_column_casing.csv"); String listName = "CaseInsensitiveList"; - String listDescription = null; - Flowable> call = uploadDataFile(file, listName, listDescription, true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, listName), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt()); @@ -615,7 +569,7 @@ public void headerCaseInsensitive() { } // Check the germplasm list - checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), listDescription, germplasmNames); + checkGermplasmList(Germplasm.constructGermplasmListName(listName, validProgram), null, germplasmNames); } @Test @@ -625,7 +579,7 @@ public void missingRequiredFieldHeaderError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -646,7 +600,7 @@ public void missingOptionalFieldHeaderError() { MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmMappingId); Flowable> call = client.exchange( POST(uploadUrl, requestBody) .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) @@ -665,12 +619,12 @@ public void missingOptionalFieldHeaderError() { public void circularParentDependencyError() { File file = new File("src/test/resources/files/germplasm_import/circular_parent_dependencies.csv"); MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.circularDependency, result.getAsJsonObject("progress").get("message").getAsString()); @@ -681,62 +635,17 @@ public void circularParentDependencyError() { public void selfReferenceParentError() { File file = new File("src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv"); MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); - Flowable> call = uploadDataFile(file, "Bad List", null,true); + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(GERM_LIST_NAME, "Bad List"), true, client, validProgram, germplasmMappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = getUploadedFile(importId); + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, validProgram, germplasmMappingId); JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt()); assertEquals(GermplasmProcessor.circularDependency, result.getAsJsonObject("progress").get("message").getAsString()); } - public Flowable> uploadDataFile(File file, String listName, String listDescription, Boolean commit) { - MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); - - // Upload file - String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", validProgram.getId(), germplasmImportId); - Flowable> call = client.exchange( - POST(uploadUrl, requestBody) - .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.OK, response.getStatus()); - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); - String importId = result.get("importId").getAsString(); - - // Process data - String url = String.format("/programs/%s/import/mappings/%s/data/%s/%s", validProgram.getId(), germplasmImportId, importId, commit ? "commit" : "preview"); - Map listBody = new HashMap<>(); - listBody.put("germplasmListName", listName); - listBody.put("germplasmListDescription", listDescription); - Flowable> processCall = client.exchange( - PUT(url, listBody) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - return processCall; - } - - public HttpResponse getUploadedFile(String importId) throws InterruptedException { - Flowable> call = client.exchange( - GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", validProgram.getId(), germplasmImportId, importId)) - .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - HttpResponse response = call.blockingFirst(); - - if (response.getStatus().equals(HttpStatus.ACCEPTED)) { - Thread.sleep(1000); - return getUploadedFile(importId); - } else { - return response; - } - - - } - public void checkBasicResponse(JsonObject germplasm, Table fileData, Integer i) { @@ -784,8 +693,8 @@ public void checkGermplasmList(String listName, String listDescription, List response = call.blockingFirst(); JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); JsonArray data = result.getAsJsonArray("data"); - Boolean nameFound = false; - Boolean descriptionFound = false; + boolean nameFound = false; + boolean descriptionFound = false; String listId = null; for (JsonElement listElement: data) { JsonObject listObject = listElement.getAsJsonObject(); @@ -827,9 +736,7 @@ public void checkGermplasmList(String listName, String listDescription, List> call = client.exchange( + POST("/programs/", gson.toJson(programRequest)) + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + return gson.fromJson(result, Program.class); + } + public Flowable> uploadDataFile(File file, Map userData, Boolean commit, RxHttpClient client, Program program, String mappingId) { + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/data/%s/%s", program.getId(), mappingId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + } + + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + + if (response.getStatus().equals(HttpStatus.ACCEPTED)) { + Thread.sleep(1000); + return getUploadedFile(importId, client, program, mappingId); + } else { + return response; + } + } + + public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, SpeciesService speciesService, UserDAO userDAO, DSLContext brapiDsl, String mappingTemplateName) { + var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + var brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Set up BrAPI + brapiDsl.execute(brapiFp.get("InsertSpecies")); + + // Species + Species validSpecies = speciesService.getAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Insert program + ProgramRequest program = ProgramRequest.builder() + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); + Program validProgram = this.insertAndFetchTestProgram(program, client, gson); + + // Get import + Flowable> call = client.exchange( + GET("/import/mappings?importName="+mappingTemplateName).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + return Map.of("program", validProgram, + "mappingId", mappingId, + "testUser", testUser, + "securityFp", securityFp); + } + + + + public JsonObject uploadAndFetch(File file, Map userData, Boolean commit, RxHttpClient client, Program program, String mappingId) throws InterruptedException { + Flowable> call = uploadDataFile(file, userData, commit, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } +} diff --git a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java index 15c3e33be..2abe8e486 100644 --- a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java @@ -17,6 +17,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import java.lang.reflect.Field; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.*; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -62,7 +64,7 @@ public ApiResponse, Optional Date: Tue, 7 Feb 2023 21:54:02 -0500 Subject: [PATCH 05/15] [BI-1195] Finishing implementation of tests for experiment import Added a migration for updating existing locations with a program xref --- .../processors/ExperimentProcessor.java | 24 ++- .../daos/ProgramLocationDAO.java | 2 +- ...V1_0_12__Update_BrAPI_Locations_XRefs.java | 124 ++++++++++++ .../impl/BreedingMethodServiceImpl.java | 6 +- .../importer/ExperimentFileImportTest.java | 190 +++++++++++++++--- .../ResponseUtilsIntegrationTest.java | 42 +++- 6 files changed, 337 insertions(+), 51 deletions(-) create mode 100644 src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 6abda1bd9..a516d35d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -516,23 +516,23 @@ private void validateObservationUnits(ValidationErrors validationErrors, Set fetchExistingObservations(List referencedTraits, Program program) throws ApiException { Set ouDbIds = new HashSet<>(); Set variableDbIds = new HashSet<>(); - Map variableNameByDbId = new HashMap<>(); - Map ouNameByDbId = new HashMap<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); Map studyNameByDbId = studyByNameNoScope.values() .stream() .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, BrAPIStudy::getStudyName)); + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); observationUnitByNameNoScope.values().forEach(ou -> { if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); } - ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), ou.getBrAPIObject()); + ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); }); for (Trait referencedTrait : referencedTraits) { variableDbIds.add(referencedTrait.getObservationVariableDbId()); - variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); } List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); @@ -540,8 +540,8 @@ private Map fetchExistingObservations(List refe return existingObservations.stream() .map(obs -> { String studyName = studyNameByDbId.get(obs.getStudyDbId()); - String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()).getObservationVariableName(); - String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()).getObservationUnitName(); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); @@ -746,6 +746,8 @@ private PendingImportObject fetchOrCreateObservationPIO(Experi newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } + newObservation.setStudyDbId(this.studyByNameNoScope.get(importRow.getEnv()).getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); } @@ -765,8 +767,10 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b if (commit) { String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season - String seasonID = this.yearToSeasonDbId(year, program.getId()); - newStudy.setSeasons(Collections.singletonList(seasonID)); + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } } pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); @@ -850,7 +854,7 @@ private void updateObservationDbIds(BrAPIObservationUnit obsUnit, String program .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.STUDY_NAME) .getAsString() - .equals(obsUnit.getStudyName()) + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getStudyName(), programKey)) && Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getBrAPIObject().getObservationUnitName(), programKey) .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(obsUnit.getObservationUnitName(), programKey)) ) diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index 605eabc60..2774f4f4b 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -143,7 +143,7 @@ private List parseRecords(List records, boolean fetchBr locationIds.add(location.getId()); } - if(fetchBrAPIObject) { + if(fetchBrAPIObject && !resultLocations.isEmpty()) { List brAPILocations = getBrapiLocations(locationIds, resultLocations.values().stream().findFirst().get().getProgramId()); if (brAPILocations.size() != resultLocations.size()) { diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java new file mode 100644 index 000000000..4629b8ec1 --- /dev/null +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java @@ -0,0 +1,124 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.db.migration; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.core.LocationQueryParams; +import org.brapi.client.v2.modules.core.LocationsApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.response.BrAPILocationListResponse; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; + +@Slf4j +public class V1_0_12__Update_BrAPI_Locations_XRefs extends BaseJavaMigration { + + final private String DEFAULT_URL_KEY = "default-url"; + final private String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; + + public void migrate(Context context) throws Exception { + Map placeholders = context.getConfiguration().getPlaceholders(); + String defaultUrl = placeholders.get(DEFAULT_URL_KEY); + String referenceSource = placeholders.get(BRAPI_REFERENCE_SOURCE_KEY); + + // Get all the programs + List programs = getAllPrograms(context, defaultUrl); + Map locationsApiForProgram = new HashMap<>(); + for (Program program : programs) { + BrAPIClient client = new BrAPIClient(program.getBrapiUrl(), 240000); + locationsApiForProgram.put(program.getId(), new LocationsApi(client)); + } + + List locations = getAllLocations(context); + // Process all the locations in the system + String locationReferenceSource = String.format("%s", referenceSource); + for(ProgramLocation location : locations) { + log.debug("Migrating BrAPI locations missing program external reference for locationId: " + location.getId()); + + String programReferenceSource = String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName()); + var api = locationsApiForProgram.get(location.getProgramId()); + + LocationQueryParams queryParams = new LocationQueryParams(); + queryParams.externalReferenceSource(locationReferenceSource); + queryParams.externalReferenceID(location.getId().toString()); + queryParams.page(0); + queryParams.pageSize(1000); + ApiResponse locationsResponse = api.locationsGet(queryParams); + + if(locationsResponse.getBody().getResult().getData().size() == 1) { + BrAPILocation brapiLocation = locationsResponse.getBody().getResult().getData().get(0); + Optional programRef = Utilities.getExternalReference(brapiLocation.getExternalReferences(), programReferenceSource); + + if(programRef.isEmpty()) { + BrAPIExternalReference programIdRef = new BrAPIExternalReference() + .referenceID(location.getProgramId().toString()) + .referenceSource(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName())); + brapiLocation.getExternalReferences().add(programIdRef); + + api.locationsLocationDbIdPut(brapiLocation.getLocationDbId(), brapiLocation); + } + } + } + log.debug("Done updating locations"); + } + + private List getAllLocations(Context context) throws SQLException { + List locations = new ArrayList<>(); + try (Statement select = context.getConnection().createStatement()) { + try (ResultSet rows = select.executeQuery("SELECT * FROM place")) { + while (rows.next()) { + ProgramLocation location = new ProgramLocation(); + location.setId(UUID.fromString(rows.getString("id"))); + location.setProgramId(UUID.fromString(rows.getString("program_id"))); + locations.add(location); + } + } + } + return locations; + } + + private List getAllPrograms(Context context, String defaultUrl) throws Exception { + List programs = new ArrayList<>(); + try (Statement select = context.getConnection().createStatement()) { + try (ResultSet rows = select.executeQuery("SELECT id, brapi_url, key FROM program where active = true ORDER BY id")) { + while (rows.next()) { + Program program = new Program(); + program.setId(UUID.fromString(rows.getString(1))); + String brapi_url = rows.getString(2); + if (brapi_url == null) brapi_url = defaultUrl; + program.setBrapiUrl(brapi_url); + program.setKey(rows.getString(3)); + programs.add(program); + } + } + } + return programs; + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 018431fc6..2756f43ce 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -49,7 +49,11 @@ public List fetchBreedingMethodsInUse(UUID programI //TODO retest with new germplasm after updating the DAO to return the correct ID for a method germplasmService.getGermplasm(programId).forEach(germplasm -> { UUID id = UUID.fromString(germplasm.getBreedingMethodDbId()); - inUse.put(id, programMethods.get(id)); + if(programMethods.containsKey(id)) { + inUse.put(id, programMethods.get(id)); + } else { + throw new IllegalStateException("Could not find breeding method by id: " + id); + } }); return new ArrayList<>(inUse.values()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index af56ea1e8..ae59f4878 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer; import com.google.gson.*; +import com.google.gson.reflect.TypeToken; import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; @@ -29,6 +30,7 @@ import lombok.SneakyThrows; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -43,10 +45,7 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.enums.DataType; @@ -138,6 +137,9 @@ public class ExperimentFileImportTest extends BrAPITest { @Inject private BrAPIObservationDAO observationDAO; + @Inject + private BrAPISeasonDAO seasonDAO; + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -204,7 +206,7 @@ public void importNewExpNewLocNoObsSuccess() { assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); - assertRowSaved(validRow, row, program, null); + assertRowSaved(validRow, program, null); } @Test @@ -253,7 +255,7 @@ public void importNewEnvExistingExpNoObsSuccess() { assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); - assertRowSaved(newEnv, row, program, null); + assertRowSaved(newEnv, program, null); } @Test @@ -341,13 +343,12 @@ public void importNewExpWithObs() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); - System.out.println(row); assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); - assertRowSaved(newExp, row, program, traits); + assertRowSaved(newExp, program, traits); } @Test @@ -466,22 +467,67 @@ public void importNewObsExisingOu() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); - System.out.println("row:" + row); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - assertRowSaved(newObservation, row, program, traits); + assertRowSaved(newObservation, program, traits); } @Test @SneakyThrows - public void importNewObsExisingOuWithExistingObs() { - fail(); + public void verifyFailureImportNewObsExisingOuWithExistingObs() { + List traits = createTraits(1); + Program program = createProgram("New Obs Existing Obs", "EXOBS", "EXOBS", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObservation = new HashMap<>(); + newObservation.put(Columns.GERMPLASM_GID, "1"); + newObservation.put(Columns.TEST_CHECK, "T"); + newObservation.put(Columns.EXP_TITLE, "Test Exp"); + newObservation.put(Columns.EXP_UNIT, "Plot"); + newObservation.put(Columns.EXP_TYPE, "Phenotyping"); + newObservation.put(Columns.ENV, "New Env"); + newObservation.put(Columns.ENV_LOCATION, "Location A"); + newObservation.put(Columns.ENV_YEAR, "2023"); + newObservation.put(Columns.EXP_UNIT_ID, "a-1"); + newObservation.put(Columns.REP_NUM, "1"); + newObservation.put(Columns.BLOCK_NUM, "1"); + newObservation.put(Columns.ROW, "1"); + newObservation.put(Columns.COLUMN, "1"); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(traits.get(0).getObservationVariableName(), "2"); + + uploadAndVerifyFailure(program, writeDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName()); } - private Map assertRowSaved(Map expected, JsonObject actual, Program program, List traits) throws ApiException { + private Map assertRowSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); List trials = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)expected.get(Columns.EXP_TITLE), program.getKey())), program); @@ -492,7 +538,14 @@ private Map assertRowSaved(Map expected, JsonObj List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program); assertFalse(studies.isEmpty()); - BrAPIStudy study = studies.get(0); + BrAPIStudy study = null; + for(BrAPIStudy s : studies) { + if(expected.get(Columns.ENV).equals(Utilities.removeProgramKeyAndUnknownAdditionalData(s.getStudyName(), program.getKey()))) { + study = s; + break; + } + } + assertNotNull(study, "Could not find study by name: " + expected.get(Columns.ENV)); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -506,33 +559,110 @@ private Map assertRowSaved(Map expected, JsonObj assertFalse(germplasms.isEmpty()); BrAPIGermplasm germplasm = germplasms.get(0); -// assertNotNull(actual.get("trial")); -// BrAPITrial trial = gson.fromJson(actual.getAsJsonObject("trial").getAsJsonObject("brAPIObject"), BrAPITrial.class); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); + + ret.put("trial", trial); + ret.put("study", study); + ret.put("location", location); + ret.put("observationUnit", ou); + ret.put("germplasm", germplasm); + + List observations = null; + if(traits != null) { + observations = observationDAO.getObservationsByStudyName(List.of(study.getStudyName()), program); + assertFalse(observations.isEmpty()); + + ret.put("observations", observations); + } + + assertNotNull(germplasm.getGermplasmName()); + assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); + if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { + assertEquals(expected.get(Columns.TEST_CHECK), + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); + } + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); + assertEquals(expected.get(Columns.EXP_DESCRIPTION), trial.getTrialDescription()); + assertEquals(expected.get(Columns.EXP_UNIT), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_UNIT), ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); + assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_LOCATION), study.getLocationName()); + assertEquals(expected.get(Columns.ENV_LOCATION), location.getName()); + assertEquals(expected.get(Columns.ENV_YEAR), season.getSeasonName()); + assertEquals(expected.get(Columns.EXP_UNIT_ID), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + BrAPIObservationUnitLevelRelationship rep = null; + BrAPIObservationUnitLevelRelationship block = null; + for (BrAPIObservationUnitLevelRelationship rel : ou.getObservationUnitPosition().getObservationLevelRelationships()) { + if ("rep".equals(rel.getLevelName()) && rep == null) { + rep = rel; + } else if ("block".equals(rel.getLevelName()) && block == null) { + block = rel; + } + } + assertNotNull(rep); + assertNotNull(block); + assertEquals(expected.get(Columns.REP_NUM), rep.getLevelCode()); + assertEquals(expected.get(Columns.BLOCK_NUM), block.getLevelCode()); + if(expected.containsKey(Columns.ROW)) { + assertEquals(expected.get(Columns.ROW), ou.getObservationUnitPosition().getPositionCoordinateX()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_ROW, ou.getObservationUnitPosition().getPositionCoordinateXType()); + } + if(expected.containsKey(Columns.COLUMN)) { + assertEquals(expected.get(Columns.COLUMN), ou.getObservationUnitPosition().getPositionCoordinateY()); + assertEquals(BrAPIPositionCoordinateTypeEnum.GRID_COL, ou.getObservationUnitPosition().getPositionCoordinateYType()); + } + if(expected.containsKey(Columns.TREATMENT_FACTORS) && StringUtils.isNotBlank((String)expected.get(Columns.TREATMENT_FACTORS))) { + assertEquals(expected.get(Columns.TREATMENT_FACTORS), ou.getTreatments().get(0).getFactor()); + } + + if(traits != null) { + List expectedVariableObservation = new ArrayList<>(); + List actualVariableObservation = new ArrayList<>(); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + for(Trait trait : traits) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); + } + + return ret; + } + + private Map assertValidPreviewRow(Map expected, JsonObject actual, Program program, List traits) { + Map ret = new HashMap<>(); + + assertNotNull(actual.get("trial")); + BrAPITrial trial = gson.fromJson(actual.getAsJsonObject("trial").getAsJsonObject("brAPIObject"), BrAPITrial.class); ret.put("trial", trial); -// assertNotNull(actual.get("study")); -// BrAPIStudy study = gson.fromJson(actual.getAsJsonObject("study").getAsJsonObject("brAPIObject"), BrAPIStudy.class); + assertNotNull(actual.get("study")); + BrAPIStudy study = gson.fromJson(actual.getAsJsonObject("study").getAsJsonObject("brAPIObject"), BrAPIStudy.class); ret.put("study", study); -// assertNotNull(actual.get("location")); -// ProgramLocation location = gson.fromJson(actual.getAsJsonObject("location").getAsJsonObject("brAPIObject"), ProgramLocation.class); + assertNotNull(actual.get("location")); + ProgramLocation location = gson.fromJson(actual.getAsJsonObject("location").getAsJsonObject("brAPIObject"), ProgramLocation.class); ret.put("location", location); -// assertNotNull(actual.get("observationUnit")); -// BrAPIObservationUnit ou = gson.fromJson(actual.getAsJsonObject("observationUnit").getAsJsonObject("brAPIObject"), BrAPIObservationUnit.class); + assertNotNull(actual.get("observationUnit")); + BrAPIObservationUnit ou = gson.fromJson(actual.getAsJsonObject("observationUnit").getAsJsonObject("brAPIObject"), BrAPIObservationUnit.class); ret.put("observationUnit", ou); -// assertNotNull(actual.get("germplasm")); -// BrAPIGermplasm germplasm = gson.fromJson(actual.getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"), BrAPIGermplasm.class); + assertNotNull(actual.get("germplasm")); + BrAPIGermplasm germplasm = gson.fromJson(actual.getAsJsonObject("germplasm").getAsJsonObject("brAPIObject"), BrAPIGermplasm.class); ret.put("germplasm", germplasm); List observations = null; if(traits != null) { - observations = observationDAO.getObservationsByStudyName(List.of(study.getStudyName()), program); - assertFalse(observations.isEmpty()); - -// assertNotNull(actual.get("observations")); -// observations = gson.fromJson(actual.get("observations"), new TypeToken>(){}.getType()); + assertNotNull(actual.get("observations")); + observations = gson.fromJson(actual.get("observations"), new TypeToken>(){}.getType()); ret.put("observations", observations); } @@ -587,7 +717,7 @@ private Map assertRowSaved(Map expected, JsonObj if(traits != null) { List expectedVariableObservation = new ArrayList<>(); List actualVariableObservation = new ArrayList<>(); - observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", observation.getObservationVariableName(), observation.getValue()))); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); for(Trait trait : traits) { expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); } diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index 38af41ae3..ef42182b5 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -29,9 +29,10 @@ import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; -import lombok.SneakyThrows; -import org.breedinginsight.DatabaseTest; +import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; @@ -42,11 +43,15 @@ import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.User; +import org.breedinginsight.services.ProgramLocationService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.jooq.DSLContext; import org.junit.jupiter.api.*; import javax.inject.Inject; - +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -57,7 +62,7 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class ResponseUtilsIntegrationTest extends DatabaseTest { +public class ResponseUtilsIntegrationTest extends BrAPITest { ProgramEntity validProgram; List locations; @@ -74,11 +79,12 @@ public class ResponseUtilsIntegrationTest extends DatabaseTest { private PlaceDao locationDao; @Inject private UserDAO userDAO; + @Inject + private ProgramLocationService locationService; // Set up program locations @BeforeAll - @SneakyThrows - public void setup() { + public void setup() throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { // Insert our traits into the db fp = FannyPack.fill("src/test/resources/sql/ResponseUtilsIntegrationTest.sql"); @@ -90,11 +96,29 @@ public void setup() { // Insert program dsl.execute(fp.get("InsertProgram")); + validProgram = programDao.findAll().get(0); // Insert program locations - dsl.execute(fp.get("InsertProgramLocations")); - - validProgram = programDao.findAll().get(0); + List newLocations = new ArrayList<>(); + newLocations.add(ProgramLocationRequest.builder() + .name("place1") + .abbreviation("abbrev1") + .slope(new BigDecimal("1.1")) + .build()); + for(int i = 2; i < 25; i++) { + newLocations.add(ProgramLocationRequest.builder() + .name("place"+i) + .abbreviation("abbrev"+i) + .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+Math.random()*10.0)) + .build()); + } + for(int i = 25; i < 31; i++) { + newLocations.add(ProgramLocationRequest.builder() + .name("place"+i) + .build()); + } + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + locationService.create(user, validProgram.getId(), newLocations); locations = locationDao.findAll(); } From 7ec6a5f8d8f4f587fa11c03357a13ba27b5d4a30 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 8 Feb 2023 16:03:45 -0500 Subject: [PATCH 06/15] [BI-1195] Appending program key to location names when saving to BrAPI server --- .../daos/ProgramLocationDAO.java | 10 +++---- .../services/ProgramLocationService.java | 27 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index 2774f4f4b..08ebe63e2 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -169,7 +169,7 @@ private List parseRecords(List records, boolean fetchBr return new ArrayList<>(resultLocations.values()); } - public void createProgramLocationBrAPI(ProgramLocation location) { + public void createProgramLocationBrAPI(ProgramLocation location, Program program) { BrAPIExternalReference locationIdRef = new BrAPIExternalReference() .referenceID(location.getId().toString()) @@ -192,7 +192,7 @@ public void createProgramLocationBrAPI(ProgramLocation location) { .externalReferences(List.of(locationIdRef, programIdRef)) //.instituteAddress() do not keep this in our model //.instituteName() do not keep this in our model - .locationName(location.getName()) + .locationName(Utilities.appendProgramKey(location.getName(), program.getKey())) //.locationType() do not keep this in our model //.siteStatus() do not keep this in our model .slope(location.getSlope() != null ? location.getSlope().toPlainString() : null) @@ -207,13 +207,13 @@ public void createProgramLocationBrAPI(ProgramLocation location) { location.setLocationDbId(brapiResponse.getBody().getResult().getData().get(0).getLocationDbId()); } } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); + log.error(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); } } - public void updateProgramLocationBrAPI(ProgramLocation location) { + public void updateProgramLocationBrAPI(ProgramLocation location, Program program) { try { List brApiLocations = getBrapiLocations(List.of(location.getId()), location.getProgramId()); @@ -237,7 +237,7 @@ public void updateProgramLocationBrAPI(ProgramLocation location) { brApiLocation.setExposure(location.getExposure()); //brApiLocation.setInstituteAddress(); do not keep this in our model //brApiLocation.setInstituteName(); do not keep this in our model - brApiLocation.setLocationName(location.getName()); + brApiLocation.setLocationName(Utilities.appendProgramKey(location.getName(), program.getKey())); //brApiLocation.setLocationType(); do not keep this in our model //brApiLocation.setSiteStatus(); do not keep this in our model brApiLocation.setSlope(location.getSlope() != null ? location.getSlope().toPlainString() : null); diff --git a/src/main/java/org/breedinginsight/services/ProgramLocationService.java b/src/main/java/org/breedinginsight/services/ProgramLocationService.java index 7aa1c504d..b7d0131af 100644 --- a/src/main/java/org/breedinginsight/services/ProgramLocationService.java +++ b/src/main/java/org/breedinginsight/services/ProgramLocationService.java @@ -25,6 +25,7 @@ import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.dao.db.tables.pojos.PlaceEntity; import org.breedinginsight.daos.ProgramLocationDAO; +import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; @@ -195,13 +196,11 @@ boolean isListPointsLatLngValid(List points) { public List create(AuthenticatedUser actingUser, UUID programId, List newLocations) throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { // check if programId exists - if (!programService.exists(programId)) { - throw new DoesNotExistException("Program id does not exist"); - } + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); List ret = new ArrayList<>(); for(ProgramLocationRequest newLoc : newLocations) { - ret.add(createLocation(actingUser, programId, newLoc)); + ret.add(createLocation(actingUser, program, newLoc)); } return ret; @@ -213,20 +212,19 @@ public ProgramLocation create(AuthenticatedUser actingUser, throws DoesNotExistException, MissingRequiredInfoException, UnprocessableEntityException { // check if programId exists - if (!programService.exists(programId)) { - throw new DoesNotExistException("Program id does not exist"); - } + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - return createLocation(actingUser, programId, programLocationRequest); + return createLocation(actingUser, program, programLocationRequest); } - private ProgramLocation createLocation(AuthenticatedUser actingUser, UUID programId, ProgramLocationRequest programLocationRequest) throws UnprocessableEntityException, MissingRequiredInfoException { + private ProgramLocation createLocation(AuthenticatedUser actingUser, Program program, ProgramLocationRequest programLocationRequest) throws UnprocessableEntityException, MissingRequiredInfoException, DoesNotExistException { // validate fields UUID countryId = validateCountryId(programLocationRequest); UUID environmentTypeId = validateEnvironmentTypeId(programLocationRequest); UUID accessibilityId = validateAccessibilityId(programLocationRequest); UUID topographyId = validateTopographyId(programLocationRequest); String coordinates = validateCoordinates(programLocationRequest); + UUID programId = program.getId(); // parse and create the program location object PlaceEntity placeEntity = PlaceEntity.builder() @@ -248,15 +246,14 @@ private ProgramLocation createLocation(AuthenticatedUser actingUser, UUID progra .build(); - ProgramLocation location = null; // Insert and update // This is warped in a transaction so if the BrAPI save call fails, the BI database insert is rolled back. - location = dsl.transactionResult(configuration -> { + ProgramLocation location = dsl.transactionResult(configuration -> { programLocationDao.insert(placeEntity); ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).get(); // Add location to brapi service - programLocationDao.createProgramLocationBrAPI(progLocation); + programLocationDao.createProgramLocationBrAPI(progLocation, program); return progLocation; }); @@ -269,9 +266,7 @@ public ProgramLocation update(AuthenticatedUser actingUser, ProgramLocationRequest programLocationRequest) throws DoesNotExistException, MissingRequiredInfoException, UnprocessableEntityException { - if (!programService.exists(programId)) { - throw new DoesNotExistException("Program id does not exist"); - } + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); PlaceEntity placeEntity = programLocationDao.fetchOneById(locationId); if (placeEntity == null || (!placeEntity.getProgramId().equals(programId))){ @@ -304,7 +299,7 @@ public ProgramLocation update(AuthenticatedUser actingUser, ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).get(); // Update location in brapi service - programLocationDao.updateProgramLocationBrAPI(progLocation); + programLocationDao.updateProgramLocationBrAPI(progLocation, program); return progLocation; }); return location; From 5ee8daaed1ef3e767745686f7f4c291adda9538b Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 9 Feb 2023 09:39:27 -0500 Subject: [PATCH 07/15] [BI-1195] Fixing failed test --- .../brapps/importer/ExperimentFileImportTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index ae59f4878..6b3d1124c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -592,8 +592,8 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.EXP_TYPE), trial.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); assertEquals(expected.get(Columns.EXP_TYPE), study.getStudyType()); assertEquals(expected.get(Columns.ENV), Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); - assertEquals(expected.get(Columns.ENV_LOCATION), study.getLocationName()); - assertEquals(expected.get(Columns.ENV_LOCATION), location.getName()); + assertEquals(expected.get(Columns.ENV_LOCATION), Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + assertEquals(expected.get(Columns.ENV_LOCATION), Utilities.removeProgramKey(location.getName(), program.getKey())); assertEquals(expected.get(Columns.ENV_YEAR), season.getSeasonName()); assertEquals(expected.get(Columns.EXP_UNIT_ID), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); From c6fe3886fab4776580a856f27537873c406ae062 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 9 Feb 2023 11:45:13 -0500 Subject: [PATCH 08/15] [BI-1195] Fixing failed test --- .../response/ResponseUtilsIntegrationTest.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index ef42182b5..fadb32447 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -40,8 +40,8 @@ import org.breedinginsight.dao.db.tables.daos.PlaceDao; import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.PlaceEntity; -import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -51,6 +51,8 @@ import org.junit.jupiter.api.*; import javax.inject.Inject; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -64,7 +66,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ResponseUtilsIntegrationTest extends BrAPITest { - ProgramEntity validProgram; + Program validProgram; List locations; private FannyPack fp; @@ -84,7 +86,7 @@ public class ResponseUtilsIntegrationTest extends BrAPITest { // Set up program locations @BeforeAll - public void setup() throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException { + public void setup() throws MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { // Insert our traits into the db fp = FannyPack.fill("src/test/resources/sql/ResponseUtilsIntegrationTest.sql"); @@ -96,7 +98,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce // Insert program dsl.execute(fp.get("InsertProgram")); - validProgram = programDao.findAll().get(0); + validProgram = new Program(programDao.findAll().get(0)); // Insert program locations List newLocations = new ArrayList<>(); @@ -118,8 +120,14 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce .build()); } AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); - locationService.create(user, validProgram.getId(), newLocations); + Method createLocationMethod = locationService.getClass() + .getDeclaredMethod("createLocation", AuthenticatedUser.class, Program.class, ProgramLocationRequest.class); + createLocationMethod.setAccessible(true); + for (ProgramLocationRequest location : newLocations) { + createLocationMethod.invoke(locationService, user, validProgram, location); + } + createLocationMethod.setAccessible(false); locations = locationDao.findAll(); } From f23c95cd163af6631e654b52615c1054287f8493 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 9 Feb 2023 12:09:05 -0500 Subject: [PATCH 09/15] [BI-1195] Fixing failed test --- .../utilities/response/ResponseUtilsIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index fadb32447..1b5543868 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -111,7 +111,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce newLocations.add(ProgramLocationRequest.builder() .name("place"+i) .abbreviation("abbrev"+i) - .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+Math.random()*10.0)) + .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+Math.random()*20.0)) .build()); } for(int i = 25; i < 31; i++) { @@ -121,6 +121,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce } AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + //accessing the private method to bypass the fetch of a program from the db/brapi (which fails bc the program isn't in the brapi server) Method createLocationMethod = locationService.getClass() .getDeclaredMethod("createLocation", AuthenticatedUser.class, Program.class, ProgramLocationRequest.class); createLocationMethod.setAccessible(true); From cc9214a4b88367b6ee9baaaacd2065e64ba07f3f Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 16 Feb 2023 14:50:53 -0500 Subject: [PATCH 10/15] [BI-1195] Updates from PR feedback --- .../brapi/v2/GermplasmController.java | 14 +-- .../brapps/importer/daos/BrAPIProgramDAO.java | 7 +- .../brapps/importer/daos/BrAPIStudyDAO.java | 6 +- .../brapps/importer/daos/BrAPITrialDAO.java | 14 +-- .../processors/ExperimentProcessor.java | 116 ++++++++---------- .../daos/ProgramLocationDAO.java | 51 +++----- .../daos/impl/UserDAOImpl.java | 12 +- .../breedinginsight/utilities/Utilities.java | 13 ++ 8 files changed, 99 insertions(+), 134 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index dae16e60c..60ee776d0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -203,12 +203,9 @@ public HttpResponse getGermplasmPedigreeInfo( metadata.setPagination(pagination); response = new BrAPIGermplasmPedigreeResponse(); } else { - Optional germplasmOptional = germplasmService.getGermplasmByDBID(programId, germplasmId); - if(germplasmOptional.isEmpty()) { - throw new DoesNotExistException("DBID for this germplasm does not exist"); - } + BrAPIGermplasm germplasm = germplasmService.getGermplasmByDBID(programId, germplasmId) + .orElseThrow(() -> new DoesNotExistException("DBID for this germplasm does not exist")); - BrAPIGermplasm germplasm = germplasmOptional.get(); //Forward the pedigree call to the backing BrAPI system of the program passing the germplasmDbId that came in the request GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); ApiResponse pedigreeResponse = api.germplasmGermplasmDbIdPedigreeGet(germplasmId, notation, includeSiblings); @@ -308,8 +305,11 @@ public HttpResponse getGermplasmProgenyInfo( } return HttpResponse.ok(progenyResponse.getBody()); } - } catch (InternalServerException | ApiException e) { - log.info(e.getMessage(), e); + } catch (InternalServerException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving pedigree node"); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java index bd3c7dcca..103585849 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java @@ -26,6 +26,7 @@ import org.brapi.v2.model.core.response.BrAPIProgramListResponse; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -61,10 +62,6 @@ public Optional getProgram(UUID programId) throws ApiException { } List programs = programsResponse.getBody().getResult().getData(); - if (programs.size() == 1) { - return Optional.of(programs.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(programs); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index 04c30a52c..c735aa4a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -53,11 +53,7 @@ public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br public Optional getStudyByName(String studyName, Program program) throws ApiException { List studies = getStudiesByName(List.of(studyName), program); - if(studies.size() == 1) { - return Optional.of(studies.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(studies); } public List getStudiesByName(List studyNames, Program program) throws ApiException { BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 8c0deb30d..c8c08a93e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -85,12 +85,7 @@ public List getTrials(UUID programId) throws ApiException, DoesNotEx trialSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); trialSearch.externalReferenceIDs(List.of(programId.toString())); - Optional optionalProgram = programService.getById(programId); - if (!optionalProgram.isPresent()) - { - throw new DoesNotExistException("Program id does not exist"); - } - Program program = optionalProgram.get(); + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); @@ -120,11 +115,8 @@ private List processExperimentsForDisplay(List trials, S public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { List trials = getTrialsByDbIds(List.of(trialDbId), program); - if(trials.size() == 1) { - return Optional.of(trials.get(0)); - } else { - return Optional.empty(); - } + + return Utilities.getSingleOptional(trials); } public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a516d35d3..37103144d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -363,19 +363,19 @@ private List verifyTraits(UUID programId, List> phenotypeCols, //TODO convert this to a ValidationError throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Ontology term(s) not found: " + String.join(COMMA_DELIMITER, differences)); - } else { - // Check that each ts column corresponds to a phenotype column - List unmatchedTimestamps = tsNames.stream() - .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) - .collect(Collectors.toList()); - if (unmatchedTimestamps.size() > 0) { - //TODO convert this to a ValidationError - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, - "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); - } + } - return filteredTraits; + // Check that each ts column corresponds to a phenotype column + List unmatchedTimestamps = tsNames.stream() + .filter(e -> !(varNames.contains(e.replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .collect(Collectors.toList()); + if (unmatchedTimestamps.size() > 0) { + //TODO convert this to a ValidationError + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, + "Timestamp column(s) lack corresponding phenotype column(s): " + String.join(COMMA_DELIMITER, unmatchedTimestamps)); } + + return filteredTraits; } private List fetchFileTraits(UUID programId, Collection varNames) { @@ -697,9 +697,9 @@ private void validateGermplasm(ExperimentObservation importRow, ValidationErrors private PendingImportObject getGidPOI(ExperimentObservation importRow) { if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { return existingGermplasmByGID.get(importRow.getGid()); - } else { - return null; } + + return null; } private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) { @@ -830,14 +830,12 @@ private void updateObservationDependencyValues(Program program) { } private List getTraitList(Program program) { - List traits = null; try { - traits = ontologyService.getTraitsByProgramId(program.getId(), true); + return ontologyService.getTraitsByProgramId(program.getId(), true); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); throw new InternalServerException(e.toString(), e); } - return traits; } // Update each ovservation's observationUnit DbId, study DbId, and germplasm DbId @@ -975,9 +973,8 @@ private Map> initializeObserva if (StringUtils.isNotBlank(row.getObsUnitID())) { if(rowByObsUnitId.containsKey(row.getObsUnitID())) { throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); - } else { - rowByObsUnitId.put(row.getObsUnitID(), row); } + rowByObsUnitId.put(row.getObsUnitID(), row); } }); @@ -987,17 +984,14 @@ private Map> initializeObserva String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); if (existingObsUnits.size() == rowByObsUnitId.size()) { existingObsUnits.forEach(brAPIObservationUnit -> { - Optional idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource); - if (idRef.isPresent()) { - ExperimentObservation row = rowByObsUnitId.get(idRef.get().getReferenceID()); - row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); - ret.put(createObservationUnitKey(row), - new PendingImportObject<>(ImportObjectState.EXISTING, - brAPIObservationUnit, - UUID.fromString(idRef.get().getReferenceID()))); - } else { - throw new InternalServerException("An ObservationUnit ID was not found in any of the external references"); - } + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + ret.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceID()))); }); } @@ -1123,11 +1117,9 @@ private Map> initializeExistingGermp } existingGermplasms.forEach(existingGermplasm -> { - Optional xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)); - if(xref.isEmpty()) { - throw new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId()); - } - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.get().getReferenceID()))); + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceID()))); }); return existingGermplasmByGID; } @@ -1139,13 +1131,11 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); - Optional xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())); - if(xref.isEmpty()) { - throw new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId()); - } + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); studyByName.put( existingStudy.getStudyName(), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.get().getReferenceID()))); + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.getReferenceID()))); } private void initializeTrialsForExistingObservationUnits(Program program, List experimentImportRows, Map> trialByName) { @@ -1156,15 +1146,15 @@ private void initializeTrialsForExistingObservationUnits(Program program, List { BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isNotBlank(existingOu.getTrialDbId()) || StringUtils.isNotBlank(existingOu.getStudyDbId())) { - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - } else { + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } }); //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId @@ -1179,14 +1169,14 @@ private void initializeTrialsForExistingObservationUnits(Program program, List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); - if (trials.size() == trialDbIds.size()) { - String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - trials.forEach(trial -> processAndCacheTrial(trial, program, trialRefSource, trialByName)); - } else { + if (trials.size() != trialDbIds.size()) { List missingIds = new ArrayList<>(trialDbIds); missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); } + + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); + trials.forEach(trial -> processAndCacheTrial(trial, program, trialRefSource, trialByName)); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1197,19 +1187,17 @@ private void initializeTrialsForExistingObservationUnits(Program program, List fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { Set trialDbIds = new HashSet<>(); List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if(studies.size() == studyDbIds.size()) { - studies.forEach(study -> { - if (StringUtils.isNotBlank(study.getTrialDbId())) { - trialDbIds.add(study.getTrialDbId()); - } else { - throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); - } - }); - } else { + if(studies.size() != studyDbIds.size()) { List missingIds = new ArrayList<>(trialDbIds); missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); } + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); return trialDbIds; } @@ -1218,11 +1206,9 @@ private void processAndCacheTrial(BrAPITrial existingTrial, Program program, Str existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); //get TrialId from existingTrial - Optional experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource); - if (experimentIDRef.isEmpty()) { - throw new InternalServerException("An Experiment ID was not found in any of the external references"); - } - UUID experimentId = UUID.fromString(experimentIDRef.get().getReferenceID()); + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceID()); trialByNameNoScope.put( existingTrial.getTrialName(), @@ -1384,7 +1370,7 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { try { season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); } catch (ApiException e) { - log.error(e.getResponseBody(), e); + log.error(Utilities.generateApiExceptionLogMessage(e), e); } Integer yearInt = (season == null) ? null : season.getYear(); return (yearInt == null) ? "" : yearInt.toString(); diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index 08ebe63e2..ecf4e8c38 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -91,11 +91,7 @@ public List getByProgramId(UUID programId) throws ApiException public Optional getById(UUID programId, UUID locationId, boolean full) throws ApiException { List locations = getByIds(programId, List.of(locationId), full); - if (locations.size() > 0){ - return Optional.of(locations.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(locations); } @@ -148,22 +144,18 @@ private List parseRecords(List records, boolean fetchBr if (brAPILocations.size() != resultLocations.size()) { throw new IllegalStateException("Did not find BrAPI Location objects for each location"); - } else { - brAPILocations.forEach(brapiLocation -> { - Optional externalReference = Utilities.getExternalReference(brapiLocation.getExternalReferences(), referenceSource); - if (externalReference.isPresent()) { - ProgramLocation location = resultLocations.get(UUID.fromString(externalReference.get() - .getReferenceID())); - if (location != null) { - location.setLocationDbId(brapiLocation.getLocationDbId()); - } else { - throw new IllegalStateException("Did not find BrAPI Location for location: " + location.getId()); - } - } else { - throw new IllegalStateException("No externalReference for brapilocation: " + brapiLocation.getLocationDbId()); - } - }); } + + brAPILocations.forEach(brapiLocation -> { + BrAPIExternalReference externalReference = Utilities.getExternalReference(brapiLocation.getExternalReferences(), referenceSource) + .orElseThrow(() -> new IllegalStateException("No externalReference for BrAPI Location: " + brapiLocation.getLocationDbId())); + + ProgramLocation location = resultLocations.get(UUID.fromString(externalReference.getReferenceID())); + if(location == null) { + throw new IllegalStateException("Did not find BrAPI Location for location: " + location.getId()); + } + location.setLocationDbId(brapiLocation.getLocationDbId()); + }); } return new ArrayList<>(resultLocations.values()); @@ -302,12 +294,9 @@ private List getProgramLocationsByBrAPISearch(UUID programId, B Map brapiLocationById = new HashMap<>(); searchResult.forEach(brAPILocation -> { - Optional xref = Utilities.getExternalReference(brAPILocation.getExternalReferences(), referenceSource); - if(xref.isPresent()) { - brapiLocationById.put(UUID.fromString(xref.get().getReferenceID()), brAPILocation); - } else { - throw new IllegalStateException(String.format("Location (by dbid): %s does not have any external references", brAPILocation.getLocationDbId())); - } + BrAPIExternalReference xref = Utilities.getExternalReference(brAPILocation.getExternalReferences(), referenceSource) + .orElseThrow(() -> new IllegalStateException(String.format("Location (by dbid): %s does not have any external references", brAPILocation.getLocationDbId()))); + brapiLocationById.put(UUID.fromString(xref.getReferenceID()), brAPILocation); }); List records = getProgramLocationsQuery() @@ -316,13 +305,13 @@ private List getProgramLocationsByBrAPISearch(UUID programId, B List programLocations = parseRecords(records, false); if(programLocations.size() != brapiLocationById.size()) { throw new IllegalStateException("Didn't find all locations by id"); - } else { - programLocations.forEach(location -> { - BrAPILocation brAPILocation = brapiLocationById.get(location.getId()); - location.setLocationDbId(brAPILocation.getLocationDbId()); - }); } + programLocations.forEach(location -> { + BrAPILocation brAPILocation = brapiLocationById.get(location.getId()); + location.setLocationDbId(brAPILocation.getLocationDbId()); + }); + return programLocations; } } diff --git a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java index 449a2a742..e179bc42d 100644 --- a/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/UserDAOImpl.java @@ -64,11 +64,7 @@ public Optional getUser(UUID id) { List programUsers = programUserDAO.getProgramUsersByUserId(id); List users = parseRecords(records, programUsers); - if (users.size() > 0){ - return Optional.of(users.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(users); } public Optional getUserByOrcId(String orcid) { @@ -78,11 +74,7 @@ public Optional getUserByOrcId(String orcid) { List programUsers = programUserDAO.getProgramUsersByOrcid(orcid); List users = parseRecords(records, programUsers); - if (users.size() > 0){ - return Optional.of(users.get(0)); - } else { - return Optional.empty(); - } + return Utilities.getSingleOptional(users); } private SelectOnConditionStep getUsersQuery(){ diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 8dbde7fad..5384a8241 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -128,4 +128,17 @@ public static Optional getExternalReference(List externalReference.getReferenceSource().equals(source)).findFirst(); } + + /** + * For a list of items, if the list has only one item, return that item, otherwise return an empty {@link Optional} + * @param items {@link List} of items + * @return Optional of type T or empty Optional + */ + public static Optional getSingleOptional(List items) { + if(items.size() == 1) { + return Optional.of(items.get(0)); + } else { + return Optional.empty(); + } + } } From ace31c90717f8fcbdcca21174492e2733af0758c Mon Sep 17 00:00:00 2001 From: timparsons Date: Tue, 28 Feb 2023 11:32:43 -0500 Subject: [PATCH 11/15] [BI-1195] Fixing error when uploading multiple new environments --- .../processors/ExperimentProcessor.java | 1 + .../importer/ExperimentFileImportTest.java | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 37103144d..1a42dc80f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -520,6 +520,7 @@ private Map fetchExistingObservations(List refe Map ouNameByDbId = new HashMap<>(); Map studyNameByDbId = studyByNameNoScope.values() .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) .map(PendingImportObject::getBrAPIObject) .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6b3d1124c..08d317f34 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -209,6 +209,72 @@ public void importNewExpNewLocNoObsSuccess() { assertRowSaved(validRow, program, null); } + @Test + @SneakyThrows + public void importNewExpMultiNewEnvNoObsSuccess() { + Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map firstEnv = new HashMap<>(); + firstEnv.put(Columns.GERMPLASM_GID, "1"); + firstEnv.put(Columns.TEST_CHECK, "T"); + firstEnv.put(Columns.EXP_TITLE, "Test Exp"); + firstEnv.put(Columns.EXP_DESCRIPTION, "Test Description"); + firstEnv.put(Columns.EXP_UNIT, "Plot"); + firstEnv.put(Columns.EXP_TYPE, "Phenotyping"); + firstEnv.put(Columns.ENV, "Test Env A"); + firstEnv.put(Columns.ENV_LOCATION, "Location A"); + firstEnv.put(Columns.ENV_YEAR, "2023"); + firstEnv.put(Columns.EXP_UNIT_ID, "a-1"); + firstEnv.put(Columns.REP_NUM, "1"); + firstEnv.put(Columns.BLOCK_NUM, "1"); + firstEnv.put(Columns.ROW, "1"); + firstEnv.put(Columns.COLUMN, "1"); + firstEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); + + Map secondEnv = new HashMap<>(); + secondEnv.put(Columns.GERMPLASM_GID, "1"); + secondEnv.put(Columns.TEST_CHECK, "T"); + secondEnv.put(Columns.EXP_TITLE, "Test Exp"); + secondEnv.put(Columns.EXP_DESCRIPTION, "Test Description"); + secondEnv.put(Columns.EXP_UNIT, "Plot"); + secondEnv.put(Columns.EXP_TYPE, "Phenotyping"); + secondEnv.put(Columns.ENV, "Test Env B"); + secondEnv.put(Columns.ENV_LOCATION, "Location B"); + secondEnv.put(Columns.ENV_YEAR, "2023"); + secondEnv.put(Columns.EXP_UNIT_ID, "b-1"); + secondEnv.put(Columns.REP_NUM, "1"); + secondEnv.put(Columns.BLOCK_NUM, "1"); + secondEnv.put(Columns.ROW, "1"); + secondEnv.put(Columns.COLUMN, "1"); + secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(2, previewRows.size()); + JsonObject firstRow = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", firstRow.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", firstRow.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", firstRow.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", firstRow.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(firstEnv, program, null); + + JsonObject secondRow = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", secondRow.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", secondRow.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", secondRow.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", secondRow.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(secondEnv, program, null); + } + @Test @SneakyThrows public void importNewEnvExistingExpNoObsSuccess() { From 79bab6b20a6237284f285114c4975d4aaa25c962 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 1 Mar 2023 23:52:35 -0500 Subject: [PATCH 12/15] [BI-1195] Renaming migration due to conflict --- ...ns_XRefs.java => V1_0_13__Update_BrAPI_Locations_XRefs.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/main/java/org/breedinginsight/db/migration/{V1_0_12__Update_BrAPI_Locations_XRefs.java => V1_0_13__Update_BrAPI_Locations_XRefs.java} (98%) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java similarity index 98% rename from src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java rename to src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java index 4629b8ec1..e4285f248 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_0_12__Update_BrAPI_Locations_XRefs.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_0_13__Update_BrAPI_Locations_XRefs.java @@ -38,7 +38,7 @@ import java.util.*; @Slf4j -public class V1_0_12__Update_BrAPI_Locations_XRefs extends BaseJavaMigration { +public class V1_0_13__Update_BrAPI_Locations_XRefs extends BaseJavaMigration { final private String DEFAULT_URL_KEY = "default-url"; final private String BRAPI_REFERENCE_SOURCE_KEY = "brapi-reference-source"; From 8e830f5be6bf24103d0e4e4a95c6a30ee19d3a4d Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 2 Mar 2023 11:08:27 -0500 Subject: [PATCH 13/15] [BI-1195] updates from PR comments --- .../brapps/importer/daos/BrAPIStudyDAO.java | 7 ++----- .../brapps/importer/model/base/Location.java | 1 - .../importer/services/processors/ExperimentProcessor.java | 6 +++--- .../importer/services/processors/LocationProcessor.java | 3 --- .../java/org/breedinginsight/daos/impl/TraitDAOImpl.java | 3 --- .../breedinginsight/services/ProgramLocationService.java | 5 ++--- 6 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index c735aa4a1..bca3ee3a3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -99,10 +99,7 @@ public List getStudiesByStudyDbId(Collection studyDbIds, Pro public Optional getStudyByDbId(String studyDbId, Program program) throws ApiException { List studies = getStudiesByStudyDbId(List.of(studyDbId), program); - if(studies.size() == 1) { - return Optional.of(studies.get(0)); - } else { - return Optional.empty(); - } + + return Utilities.getSingleOptional(studies); } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java index 5df11586e..fefbe993b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Location.java @@ -19,7 +19,6 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import org.brapi.v2.model.core.BrAPILocation; import org.breedinginsight.brapps.importer.model.config.ImportFieldMetadata; import org.breedinginsight.brapps.importer.model.config.ImportFieldType; import org.breedinginsight.brapps.importer.model.config.ImportFieldTypeEnum; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 1a42dc80f..3eabe8c4d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -316,7 +316,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program } } - private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) throws MissingRequiredInfoException { + private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -1007,7 +1007,7 @@ private Map> initializeTrialByNameNoScop Map> trialByName = new HashMap<>(); String programKey = program.getKey(); - initializeTrialsForExistingObservationUnits(program, experimentImportRows, trialByName); + initializeTrialsForExistingObservationUnits(program, trialByName); List uniqueTrialNames = experimentImportRows.stream() .filter(row -> StringUtils.isBlank(row.getObsUnitID())) @@ -1139,7 +1139,7 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.getReferenceID()))); } - private void initializeTrialsForExistingObservationUnits(Program program, List experimentImportRows, Map> trialByName) { + private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { if(observationUnitByNameNoScope.size() > 0) { Set trialDbIds = new HashSet<>(); Set studyDbIds = new HashSet<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index 16b302606..cd2debb1e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -20,10 +20,8 @@ import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPILocation; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; -import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Location; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; @@ -36,7 +34,6 @@ import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.ValidatorException; -import org.breedinginsight.utilities.Utilities; import tech.tablesaw.api.Table; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index ff1d8be91..b901717ca 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -29,7 +29,6 @@ import org.brapi.client.v2.BrAPIClient; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.phenotype.VariableQueryParams; -import org.brapi.client.v2.modules.core.LocationsApi; import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.*; @@ -40,7 +39,6 @@ import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.TraitDao; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.dao.db.tables.records.TraitRecord; import org.breedinginsight.daos.ObservationDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.TraitDAO; @@ -63,7 +61,6 @@ import java.util.stream.Stream; import static org.breedinginsight.dao.db.Tables.*; -import static org.breedinginsight.services.brapi.BrAPIClientType.PHENO; import static org.jooq.impl.DSL.lower; @Singleton diff --git a/src/main/java/org/breedinginsight/services/ProgramLocationService.java b/src/main/java/org/breedinginsight/services/ProgramLocationService.java index b7d0131af..2191b597b 100644 --- a/src/main/java/org/breedinginsight/services/ProgramLocationService.java +++ b/src/main/java/org/breedinginsight/services/ProgramLocationService.java @@ -248,16 +248,15 @@ private ProgramLocation createLocation(AuthenticatedUser actingUser, Program pro // Insert and update // This is warped in a transaction so if the BrAPI save call fails, the BI database insert is rolled back. - ProgramLocation location = dsl.transactionResult(configuration -> { + return dsl.transactionResult(configuration -> { programLocationDao.insert(placeEntity); - ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).get(); + ProgramLocation progLocation = programLocationDao.getById(programId, placeEntity.getId(), false).orElseThrow(() -> new IllegalStateException("Location appears to not have been created")); // Add location to brapi service programLocationDao.createProgramLocationBrAPI(progLocation, program); return progLocation; }); - return location; } public ProgramLocation update(AuthenticatedUser actingUser, From f33d54e3d2bb0974def4ec3396101000f3cbeda8 Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 3 Mar 2023 16:35:28 -0500 Subject: [PATCH 14/15] [BI-1195] Checking that list parameters have values before executing a BrAPI search request Returning an empty list if list parameters are empty --- .../brapps/importer/daos/BrAPIListDAO.java | 8 ++- .../importer/daos/BrAPILocationDAO.java | 9 ++- .../importer/daos/BrAPIObservationDAO.java | 8 ++- .../daos/BrAPIObservationUnitDAO.java | 15 +++- .../daos/BrAPIObservationVariableDAO.java | 5 ++ .../brapps/importer/daos/BrAPIStudyDAO.java | 13 +++- .../brapps/importer/daos/BrAPITrialDAO.java | 10 ++- .../breedinginsight/daos/ObservationDAO.java | 8 ++- .../daos/ProgramLocationDAO.java | 39 +++++++---- .../daos/impl/TraitDAOImpl.java | 6 +- .../services/ProgramLocationService.java | 2 +- .../geno/impl/GigwaGenotypeServiceImpl.java | 8 +-- .../importer/ExperimentFileImportTest.java | 68 +++++++++++++++++++ 13 files changed, 170 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java index d04b05b43..4a7eb90e5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java @@ -21,7 +21,9 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.validation.constraints.NotNull; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -42,6 +44,10 @@ public BrAPIListDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brA } public List getListByName(List listNames, UUID programId) throws ApiException { + if(listNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIListSearchRequest listSearch = new BrAPIListSearchRequest(); listSearch.listNames(listNames); ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); @@ -58,7 +64,7 @@ public BrAPIListsSingleResponse getListById(String listId, UUID programId) throw return response.getBody(); } - public List getListByTypeAndExternalRef(BrAPIListTypes listType, UUID programId, String externalReferenceSource, UUID externalReferenceId) throws ApiException { + public List getListByTypeAndExternalRef(@NotNull BrAPIListTypes listType, UUID programId, String externalReferenceSource, UUID externalReferenceId) throws ApiException { BrAPIListSearchRequest searchRequest = new BrAPIListSearchRequest() .externalReferenceIDs(List.of(externalReferenceId.toString())) .externalReferenceSources(List.of(externalReferenceSource)) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java index 68febcd3b..2fe7b8b01 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java @@ -47,6 +47,9 @@ public BrAPILocationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil } public List getLocationsByName(List locationNames, UUID programId) throws ApiException { + if(locationNames.isEmpty()) { + return Collections.emptyList(); + } BrAPILocationSearchRequest locationSearchRequest = new BrAPILocationSearchRequest(); locationSearchRequest.setLocationNames(new ArrayList<>(locationNames)); @@ -65,10 +68,14 @@ public List createBrAPILocations(List brAPILocatio } public List getLocationsByDbId(Collection locationDbIds, UUID programId) throws ApiException { + if(locationDbIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPILocationSearchRequest locationSearchRequest = new BrAPILocationSearchRequest(); locationSearchRequest.setLocationDbIds(new ArrayList<>(locationDbIds)); //TODO: Locations don't connect to programs. How to get locations for the program? - LocationsApi api = new LocationsApi(programDAO.getCoreClient(programId)); + LocationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); return brAPIDAOUtil.search( api::searchLocationsPost, api::searchLocationsSearchResultsDbIdGet, diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 79464c03c..ecaa4bff0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -54,6 +54,9 @@ public BrAPIObservationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOU } public List getObservationsByStudyName(List studyNames, Program program) throws ApiException { + if(studyNames.isEmpty()) { + return Collections.emptyList(); + } BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); @@ -67,12 +70,15 @@ public List getObservationsByStudyName(List studyNames } public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { + if(ouDbIds.isEmpty() || variableDbIds.isEmpty()) { + return Collections.emptyList(); + } BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); observationSearchRequest.setObservationUnitDbIds(new ArrayList<>(ouDbIds)); observationSearchRequest.setObservationVariableDbIds(new ArrayList<>(variableDbIds)); - ObservationsApi api = new ObservationsApi(programDAO.getCoreClient(program.getId())); + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); return brAPIDAOUtil.search( api::searchObservationsPost, (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index a83a4c9d1..59c2a3041 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import javax.validation.constraints.NotNull; import java.util.*; @Singleton @@ -74,6 +75,10 @@ public List getObservationUnitsByNameAndStudyName(List getObservationUnitByName(List observationUnitNames, Program program) throws ApiException { + if(observationUnitNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); observationUnitSearchRequest.observationUnitNames(observationUnitNames); @@ -91,25 +96,29 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { + if(observationUnitExternalIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() .getProgramDbId())); observationUnitSearchRequest.externalReferenceIDs(new ArrayList<>(observationUnitExternalIds)); observationUnitSearchRequest.externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))); - ObservationUnitsApi api = new ObservationUnitsApi(programDAO.getCoreClient(program.getId())); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); return brAPIDAOUtil.search(api::searchObservationunitsPost, api::searchObservationunitsSearchResultsDbIdGet, observationUnitSearchRequest); } - public List getObservationUnitsForStudyDbId(String studyDbId, Program program) throws ApiException { + public List getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException { BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() .getProgramDbId())); observationUnitSearchRequest.studyDbIds(List.of(studyDbId)); - ObservationUnitsApi api = new ObservationUnitsApi(programDAO.getCoreClient(program.getId())); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); return brAPIDAOUtil.search(api::searchObservationunitsPost, api::searchObservationunitsSearchResultsDbIdGet, observationUnitSearchRequest); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java index 75ebe3502..86731a0f2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java @@ -26,6 +26,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -44,6 +45,10 @@ public BrAPIObservationVariableDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOU } public List getVariableByName(List variableNames, UUID programId) throws ApiException { + if(variableNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIObservationVariableSearchRequest variableSearch = new BrAPIObservationVariableSearchRequest(); variableSearch.observationVariableNames(variableNames); ObservationVariablesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationVariablesApi.class); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index bca3ee3a3..456c7adf5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.inject.Singleton; +import javax.validation.constraints.NotNull; import java.util.*; @Singleton @@ -56,6 +57,10 @@ public Optional getStudyByName(String studyName, Program program) th return Utilities.getSingleOptional(studies); } public List getStudiesByName(List studyNames, Program program) throws ApiException { + if(studyNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.studyNames(studyNames); @@ -67,7 +72,7 @@ public List getStudiesByName(List studyNames, Program progra ); } - public List getStudiesByExperimentID(UUID experimentID, Program program ) throws ApiException { + public List getStudiesByExperimentID(@NotNull UUID experimentID, Program program ) throws ApiException { BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.addExternalReferenceIDsItem(experimentID.toString()); @@ -86,10 +91,14 @@ public List createBrAPIStudies(List brAPIStudyList, UUID } public List getStudiesByStudyDbId(Collection studyDbIds, Program program) throws ApiException { + if(studyDbIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.studyDbIds(new ArrayList<>(studyDbIds)); - StudiesApi api = new StudiesApi(programDAO.getCoreClient(program.getId())); + StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); return brAPIDAOUtil.search( api::searchStudiesPost, api::searchStudiesSearchResultsDbIdGet, diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index c8c08a93e..44b1b9ba3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -57,6 +57,10 @@ public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br } public List getTrialsByName(List trialNames, Program program) throws ApiException { + if(trialNames.isEmpty()) { + return Collections.emptyList(); + } + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); trialSearch.trialNames(trialNames); @@ -120,10 +124,14 @@ public Optional getTrialByDbId(String trialDbId, Program program) th } public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { + if(trialDbIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); trialSearch.trialDbIds(new ArrayList<>(trialDbIds)); - TrialsApi api = new TrialsApi(programDAO.getCoreClient(program.getId())); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); return brAPIDAOUtil.search( api::searchTrialsPost, api::searchTrialsSearchResultsDbIdGet, diff --git a/src/main/java/org/breedinginsight/daos/ObservationDAO.java b/src/main/java/org/breedinginsight/daos/ObservationDAO.java index d15a49c28..9ecfb0588 100644 --- a/src/main/java/org/breedinginsight/daos/ObservationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ObservationDAO.java @@ -36,8 +36,8 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.UUID; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -76,6 +76,9 @@ public List getObservationsByVariableDbId(String observationVa // search by ObservationVariableDbIds public List getObservationsByVariableDbIds(List observationVariableDbIds, UUID programId) { + if(observationVariableDbIds.isEmpty()) { + return Collections.emptyList(); + } try { BrAPIObservationSearchRequest request = new BrAPIObservationSearchRequest() @@ -95,6 +98,9 @@ public List getObservationsByVariableDbIds(List observ } public List getObservationsByVariableAndBrAPIProgram(String brapiProgramId, UUID programId, List observationVariableDbIds) { + if(observationVariableDbIds.isEmpty()) { + return Collections.emptyList(); + } try { BrAPIObservationSearchRequest request = new BrAPIObservationSearchRequest() diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index ecf4e8c38..7ba7ff1f5 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -37,6 +37,7 @@ import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.PlaceDao; import org.breedinginsight.model.*; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -62,9 +63,10 @@ public class ProgramLocationDAO extends PlaceDao { private final BrAPIDAOUtil brAPIDAOUtil; private final ProgramDAO programDAO; + private final BrAPIEndpointProvider brAPIEndpointProvider; @Inject - public ProgramLocationDAO(Configuration config, DSLContext dsl, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO) { + public ProgramLocationDAO(Configuration config, DSLContext dsl, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider) { super(config); this.dsl = dsl; this.gson = new GsonBuilder() @@ -73,6 +75,7 @@ public ProgramLocationDAO(Configuration config, DSLContext dsl, @Property(name = this.referenceSource = referenceSource; this.brAPIDAOUtil = brAPIDAOUtil; this.programDAO = programDAO; + this.brAPIEndpointProvider = brAPIEndpointProvider; } // get all active locations by program id @@ -98,7 +101,8 @@ public Optional getById(UUID programId, UUID locationId, boolea public List getByIds(UUID programId, Collection locationIds, boolean full) throws ApiException { List records = getProgramLocationsQuery() - .where(PLACE.ID.in(locationIds).and(PLACE.PROGRAM_ID.eq(programId))) + .where(PLACE.ID.in(locationIds)) + .and(PLACE.PROGRAM_ID.eq(programId)) .fetch(); return parseRecords(records, full); @@ -193,7 +197,7 @@ public void createProgramLocationBrAPI(ProgramLocation location, Program program // POST locations to each brapi service // TODO: If there is a failure after the first brapi service, roll back all before the failure. try { - LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(location.getProgramId())); + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(location.getProgramId()), LocationsApi.class); ApiResponse brapiResponse = locationsAPI.locationsPost(List.of(brApiLocation)); if(brapiResponse.getBody().getResult().getData().size() == 1) { location.setLocationDbId(brapiResponse.getBody().getResult().getData().get(0).getLocationDbId()); @@ -235,7 +239,7 @@ public void updateProgramLocationBrAPI(ProgramLocation location, Program program brApiLocation.setSlope(location.getSlope() != null ? location.getSlope().toPlainString() : null); brApiLocation.setTopography(location.getTopography() != null ? location.getTopography().getName() : null); - LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(location.getProgramId())); + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(location.getProgramId()), LocationsApi.class); locationsAPI.locationsLocationDbIdPut(brApiLocation.getLocationDbId(), brApiLocation); } catch (ApiException e) { log.warn(Utilities.generateApiExceptionLogMessage(e)); @@ -244,13 +248,17 @@ public void updateProgramLocationBrAPI(ProgramLocation location, Program program } private List getBrapiLocations(List locationIds, UUID programId) throws ApiException { + if(locationIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() .externalReferenceIDs(locationIds.stream().map(UUID::toString).collect(Collectors.toList())) .externalReferenceSources(List.of(referenceSource)); // Location goes in all of the clients // TODO: If there is a failure, roll back all before the failure. - LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(programId)); + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); // Get existing brapi location return brAPIDAOUtil.search(locationsAPI::searchLocationsPost, locationsAPI::searchLocationsSearchResultsDbIdGet, searchRequest); @@ -270,6 +278,10 @@ private BrApiGeoJSON getClientGeoJson(ProgramLocation location) { public List getByDbIds(Collection locationDbIds, UUID programId) throws ApiException { + if(locationDbIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() .locationDbIds(new ArrayList<>(locationDbIds)) .externalReferenceIDs(List.of(programId.toString())) @@ -278,18 +290,21 @@ public List getByDbIds(Collection locationDbIds, UUID p return getProgramLocationsByBrAPISearch(programId, searchRequest); } - public List getByNames(List names, UUID programId) throws ApiException { - BrAPILocationSearchRequest searchRequest = new BrAPILocationSearchRequest() - .locationNames(new ArrayList<>(names)) - .externalReferenceIDs(List.of(programId.toString())) - .externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.PROGRAMS.getName()))); + public List getByNames(List names, UUID programId, boolean full) throws ApiException { + if(names.isEmpty()) { + return Collections.emptyList(); + } - return getProgramLocationsByBrAPISearch(programId, searchRequest); + List records = getProgramLocationsQuery().where(PLACE.NAME.in(names)) + .and(PLACE.PROGRAM_ID.eq(programId)) + .fetch(); + + return parseRecords(records, full); } @NotNull private List getProgramLocationsByBrAPISearch(UUID programId, BrAPILocationSearchRequest searchRequest) throws ApiException { - LocationsApi locationsAPI = new LocationsApi(programDAO.getCoreClient(programId)); + LocationsApi locationsAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), LocationsApi.class); List searchResult = brAPIDAOUtil.search(locationsAPI::searchLocationsPost, locationsAPI::searchLocationsSearchResultsDbIdGet, searchRequest); Map brapiLocationById = new HashMap<>(); diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index b901717ca..2dba86807 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -311,7 +311,9 @@ public List getObservationsForTraitsByBrAPIProgram(String brap @Override public List searchVariables(List variableIds, UUID programId) { - if (variableIds == null || variableIds.size() == 0) return new ArrayList<>(); + if (variableIds == null || variableIds.size() == 0) { + return Collections.emptyList(); + } try { BrAPIObservationVariableSearchRequest request = new BrAPIObservationVariableSearchRequest() .externalReferenceIDs(variableIds); @@ -440,7 +442,7 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra // TODO: If there is a failure after the first brapi service, roll back all before the failure. ApiResponse createdVariables = null; try { - ObservationVariablesApi variablesAPI = new ObservationVariablesApi(programDAO.getCoreClient(program.getId())); + ObservationVariablesApi variablesAPI = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationVariablesApi.class); createdVariables = variablesAPI.variablesPost(brApiVariables); } catch (ApiException e) { log.warn(Utilities.generateApiExceptionLogMessage(e)); diff --git a/src/main/java/org/breedinginsight/services/ProgramLocationService.java b/src/main/java/org/breedinginsight/services/ProgramLocationService.java index 2191b597b..8ebd85959 100644 --- a/src/main/java/org/breedinginsight/services/ProgramLocationService.java +++ b/src/main/java/org/breedinginsight/services/ProgramLocationService.java @@ -337,6 +337,6 @@ public List getLocationsByDbId(Collection locationDbIds } public List getLocationsByName(List names, UUID programId) throws ApiException { - return programLocationDao.getByNames(names, programId); + return programLocationDao.getByNames(names, programId, true); } } diff --git a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java index fc975d702..73fb4c128 100644 --- a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java @@ -387,7 +387,7 @@ private List fetchSamples(BrAPIClient genoBrAPIClient, Program prog log.debug("fetching samples for OUs"); if(observationUnits.isEmpty()) { log.debug("No OUs were supplied, returning"); - return new ArrayList<>(); + return Collections.emptyList(); } SamplesApi samplesApi = brAPIEndpointProvider.get(genoBrAPIClient, SamplesApi.class); @@ -451,7 +451,7 @@ private List fetchCallsets(BrAPIClient genoBrAPIClient, List(); + return Collections.emptyList(); } CallSetsApi callSetsApi = brAPIEndpointProvider.get(genoBrAPIClient, CallSetsApi.class); @@ -466,7 +466,7 @@ private List fetchCalls(BrAPIClient genoBrAPIClient, List(); + return Collections.emptyList(); } CallsApi callsApi = brAPIEndpointProvider.get(genoBrAPIClient, CallsApi.class); @@ -480,7 +480,7 @@ private List fetchVariants(BrAPIClient genoBrAPIClient, List(); + return Collections.emptyList(); } List variantIds = calls.stream() .map(BrAPICall::getVariantDbId) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 08d317f34..2e3295a0a 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -593,6 +593,74 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs() { uploadAndVerifyFailure(program, writeDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName()); } + /* + Scenario: + - an experiment was created with observations + - a new experiment is created after the first experiment + - verify the second experiment gets created successfully + */ + @Test + @SneakyThrows + public void importSecondExpAfterFirstExpWithObs() { + List traits = createTraits(1); + Program program = createProgram("New Exp After First", "NEAF", "NEAF", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExpA = new HashMap<>(); + newExpA.put(Columns.GERMPLASM_GID, "1"); + newExpA.put(Columns.TEST_CHECK, "T"); + newExpA.put(Columns.EXP_TITLE, "Test Exp A"); + newExpA.put(Columns.EXP_UNIT, "Plot"); + newExpA.put(Columns.EXP_TYPE, "Phenotyping"); + newExpA.put(Columns.ENV, "New Env"); + newExpA.put(Columns.ENV_LOCATION, "Location A"); + newExpA.put(Columns.ENV_YEAR, "2023"); + newExpA.put(Columns.EXP_UNIT_ID, "a-1"); + newExpA.put(Columns.REP_NUM, "1"); + newExpA.put(Columns.BLOCK_NUM, "1"); + newExpA.put(Columns.ROW, "1"); + newExpA.put(Columns.COLUMN, "1"); + newExpA.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject resultA = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + + JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRowsA.size()); + JsonObject rowA = previewRowsA.get(0).getAsJsonObject(); + + assertEquals("NEW", rowA.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", rowA.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", rowA.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", rowA.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExpA, program, traits); + + Map newExpB = new HashMap<>(); + newExpB.put(Columns.GERMPLASM_GID, "1"); + newExpB.put(Columns.TEST_CHECK, "T"); + newExpB.put(Columns.EXP_TITLE, "Test Exp B"); + newExpB.put(Columns.EXP_UNIT, "Plot"); + newExpB.put(Columns.EXP_TYPE, "Phenotyping"); + newExpB.put(Columns.ENV, "New Env"); + newExpB.put(Columns.ENV_LOCATION, "Location A"); + newExpB.put(Columns.ENV_YEAR, "2023"); + newExpB.put(Columns.EXP_UNIT_ID, "a-1"); + newExpB.put(Columns.REP_NUM, "1"); + newExpB.put(Columns.BLOCK_NUM, "1"); + newExpB.put(Columns.ROW, "1"); + newExpB.put(Columns.COLUMN, "1"); + newExpB.put(traits.get(0).getObservationVariableName(), "1"); + + JsonObject resultB = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + + JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRowsB.size()); + JsonObject rowB = previewRowsB.get(0).getAsJsonObject(); + + assertEquals("NEW", rowB.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", rowB.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", rowB.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", rowB.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExpB, program, traits); + } + private Map assertRowSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); From 004e9d09cd7a7fd34abf70043c14f73dbd246903 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 6 Mar 2023 21:44:34 -0500 Subject: [PATCH 15/15] [BI-1195] Fixing bug when saving observations --- .../imports/experimentObservation/ExperimentObservation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index fa6cc749f..d429ee126 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -293,7 +293,7 @@ public BrAPIObservation constructBrAPIObservation( observation.putAdditionalInfoItem(BrAPIAdditionalInfoFields.STUDY_NAME, getEnv()); } observation.setObservationVariableName(variableName); - observation.setObservationDbId(obsUnit.getObservationUnitDbId()); + observation.setObservationUnitDbId(obsUnit.getObservationUnitDbId()); observation.setObservationUnitName(obsUnit.getObservationUnitName()); observation.setValue(value);