diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 55a2c95f1..8503b406e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -40,4 +40,5 @@ public final class BrAPIAdditionalInfoFields { public static final String EXPERIMENT_NUMBER = "experimentNumber"; public static final String ENVIRONMENT_NUMBER = "environmentNumber"; public static final String STUDY_NAME = "studyName"; + public static final String OBSERVATION_DATASET_ID = "observationDatasetId"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java index 4a7eb90e5..a919fcf8d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java @@ -12,7 +12,7 @@ import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.request.BrAPIListSearchRequest; -import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; +import org.brapi.v2.model.core.response.*; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; @@ -97,11 +97,52 @@ private List processListsForProgram(List pro } return filteredLists; } + public List updateBrAPIList(String brAPIListDbId, BrAPIListDetails mutatedList, UUID programId) throws ApiException { + ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setAdditionalInfo(mutatedList.getAdditionalInfo()); + request.setDateCreated(mutatedList.getDateCreated()); + request.setDateModified(mutatedList.getDateModified()); + request.setExternalReferences(mutatedList.getExternalReferences()); + request.setListDescription(mutatedList.getListDescription()); + request.setListName(mutatedList.getListName()); + request.setListOwnerName(mutatedList.getListOwnerName()); + request.setListOwnerPersonDbId(mutatedList.getListOwnerPersonDbId()); + request.setListSize(mutatedList.getListSize()); + request.setListSource(mutatedList.getListSource()); + request.setListType(mutatedList.getListType()); + request.setData(mutatedList.getData()); + + // Do manually, it doesn't like List to List for some reason + ApiResponse response; + try { + response = api.listsListDbIdPut(brAPIListDbId, request); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e)); + throw e; + } + if(response != null) { + BrAPIListsSingleResponse body = response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); + } + BrAPIListDetails result = body.getResult(); + if (result == null) { + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); + } + if (result.getData() == null) { + throw new ApiException("Response result is missing data", 0, response.getHeaders(), response.getBody().toString()); + } + return result.getData(); + } + + throw new ApiException("No response after creating list"); + } - public List createBrAPILists(List brapiLists, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPILists(List brapiLists, UUID programId, ImportUpload upload) throws ApiException { ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); // Do manually, it doesn't like List to List for some reason - ApiResponse response; + ApiResponse response; try { response = api.listsPost(brapiLists); } catch (ApiException e) { @@ -109,16 +150,16 @@ public List createBrAPILists(List brapiLi throw e; } if(response != null) { - BrAPIResponse body = (BrAPIResponse) response.getBody(); + BrAPIResponse body = response.getBody(); if (body == null) { - throw new ApiException("Response is missing body"); + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); } - BrAPIResponseResult result = (BrAPIResponseResult) body.getResult(); + BrAPIResponseResult result = body.getResult(); if (result == null) { - throw new ApiException("Response body is missing result"); + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); } if (result.getData() == null) { - throw new ApiException("Response result is missing data"); + throw new ApiException("Response result is missing data", 0, response.getHeaders(), response.getBody().toString()); } return result.getData(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 44b1b9ba3..742afd2a3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -21,6 +21,9 @@ import org.brapi.client.v2.modules.core.TrialsApi; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.core.response.BrAPIListResponse; +import org.brapi.v2.model.core.response.BrAPITrialSingleResponse; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; @@ -76,7 +79,10 @@ public List createBrAPITrials(List brAPITrialList, UUID TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); return brAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); } - + public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException { + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + return brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); + } /** * Fetch formatted trials/experiments for this program * @param programId diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index d429ee126..20aade1ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -22,6 +22,7 @@ import lombok.Setter; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.config.*; @@ -197,6 +198,23 @@ public BrAPIStudy constructBrAPIStudy( return study; } + public BrAPIListDetails constructDatasetDetails( + String name, + UUID datasetId, + String referenceSourceBase, + Program program, String trialId) { + BrAPIListDetails dataSetDetails = new BrAPIListDetails(); + dataSetDetails.setListName(name); + dataSetDetails.setListType(BrAPIListTypes.OBSERVATIONVARIABLES); + dataSetDetails.setData(new ArrayList<>()); + dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); + List refs = new ArrayList<>(); + addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); + addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); + addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); + dataSetDetails.setExternalReferences(refs); + return dataSetDetails; + } public BrAPIObservationUnit constructBrAPIObservationUnit( Program program, String seqVal, @@ -340,8 +358,7 @@ private List getObsUnitExternalReferences( private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { - BrAPIExternalReference reference; - reference = new BrAPIExternalReference(); + BrAPIExternalReference reference = new BrAPIExternalReference(); reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); reference.setReferenceID(uuid.toString()); refs.add(reference); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportObjectState.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportObjectState.java index 1e646189b..5716d62f7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportObjectState.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/ImportObjectState.java @@ -19,5 +19,6 @@ public enum ImportObjectState { NEW, - EXISTING + EXISTING, + MUTATED } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java index c7619df80..9b0027e27 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java @@ -38,4 +38,5 @@ public PendingImportObject(ImportObjectState state, T brAPIObject, UUID id) { public PendingImportObject(ImportObjectState state, T brAPIObject) { this(state, brAPIObject, UUID.randomUUID()); } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java index 6ea4c0b93..aa2fd1d75 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java @@ -8,6 +8,7 @@ public enum ExternalReferenceSource { TRIALS("trials"), STUDIES("studies"), OBSERVATION_UNITS("observationunits"), + DATASET("dataset"), LISTS("lists"); private String name; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e94ab7503..c18bbc3b3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -26,9 +26,9 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPISeason; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -103,6 +103,7 @@ public class ExperimentProcessor implements Processor { private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPISeasonDAO brAPISeasonDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private final BrAPIListDAO brAPIListDAO; private final OntologyService ontologyService; private final FileMappingUtil fileMappingUtil; @@ -117,6 +118,7 @@ public class ExperimentProcessor implements Processor { private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; + private Map> obsVarDatasetByName = null; // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; @@ -138,7 +140,7 @@ public ExperimentProcessor(DSLContext dsl, BrAPIObservationDAO brAPIObservationDAO, BrAPISeasonDAO brAPISeasonDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, - OntologyService ontologyService, + BrAPIListDAO brAPIListDAO, OntologyService ontologyService, FileMappingUtil fileMappingUtil) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; @@ -148,6 +150,7 @@ public ExperimentProcessor(DSLContext dsl, this.brAPIObservationDAO = brAPIObservationDAO; this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; + this.brAPIListDAO = brAPIListDAO; this.ontologyService = ontologyService; this.fileMappingUtil = fileMappingUtil; } @@ -174,6 +177,7 @@ public void getExistingBrapiData(List importRows, Program program) this.trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); this.studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); + this.obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); this.existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); } @@ -220,7 +224,7 @@ public Map process( } // add "New" pending data to the BrapiData objects - initNewBrapiData(importRows, phenotypeCols, program, user, commit); + initNewBrapiData(importRows, phenotypeCols, program, user, referencedTraits, commit); prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); @@ -245,6 +249,9 @@ public void postBrapiData(Map mappedBrAPIImport, Program log.debug("starting post of experiment data to BrAPI server"); List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + Map mutatedTrialsById = ProcessorData + .getMutationsByObjectId(trialByNameNoScope, trial -> trial.getTrialDbId()); + List newLocations = ProcessorData.getNewObjects(this.locationByName) .stream() .map(location -> ProgramLocationRequest.builder() @@ -252,6 +259,19 @@ public void postBrapiData(Map mappedBrAPIImport, Program .build()) .collect(Collectors.toList()); List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + + List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + Map datasetNewDataById = ProcessorData + .getMutationsByObjectId(obsVarDatasetByName, listDetails -> listDetails.getListDbId()); + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); // filter out observations with no 'value' so they will not be saved List newObservations = ProcessorData.getNewObjects(this.observationByHash) @@ -271,6 +291,12 @@ public void postBrapiData(Map mappedBrAPIImport, Program .setTrialDbId(createdTrial.getTrialDbId()); } + + List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); + createdDatasets.forEach(summary -> { + obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); + }); + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); // set the DbId to the for each newly created trial for (ProgramLocation createdLocation : createdLocations) { @@ -307,7 +333,6 @@ public void postBrapiData(Map mappedBrAPIImport, Program updateObservationDependencyValues(program); brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); - log.debug("experiment import complete"); } catch (ApiException e) { log.error("Error saving experiment import: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException("Error saving experiment import", e); @@ -315,6 +340,40 @@ public void postBrapiData(Map mappedBrAPIImport, Program log.error("Error saving experiment import", e); throw new InternalServerException(e.getMessage(), e); } + + mutatedTrialsById.forEach((id, trial) -> { + try { + brapiTrialDAO.updateBrAPITrial(id, trial, program.getId()); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + + datasetNewDataById.forEach((id, dataset) -> { + try { + List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); + List newObsVarIds = dataset + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + dataset.setData(obsVarIds); + brAPIListDAO.updateBrAPIList(id, dataset, program.getId()); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + log.debug("experiment import complete"); + } private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { @@ -397,7 +456,7 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, boolean commit) { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -426,6 +485,10 @@ private void initNewBrapiData(List importRows, List> phen .getAsString(); } + if (commit) { + fetchOrCreateDatasetPIO(importRow, program, referencedTraits); + } + fetchOrCreateLocationPIO(importRow); PendingImportObject studyPIO = fetchOrCreateStudyPIO(program, commit, expSeqValue, importRow, envNextVal); @@ -755,6 +818,48 @@ private PendingImportObject fetchOrCreateObservationPIO(Experi } return pio; } + private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { + BrAPIListDetails details = pio.getBrAPIObject(); + referencedTraits.forEach(trait -> { + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { + details.getData().add(id); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { + details.getData().add(id); + pio.setState(ImportObjectState.MUTATED); + } + }); + } + private PendingImportObject fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { + PendingImportObject pio; + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + return pio; + } private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { PendingImportObject pio; @@ -1034,24 +1139,20 @@ private Map> initializeStudyByNameNoScop } List existingStudies; - Optional expTitle = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .findFirst(); - if(expTitle.isEmpty()) { - expTitle = trialByNameNoScope.keySet().stream().findFirst(); - } - PendingImportObject trial = this.trialByNameNoScope.get(expTitle.get()); - UUID experimentId = trial.getId(); + Optional> trial = getTrialPIO(experimentImportRows); try { + if (trial.isEmpty()) { + // TODO: throw ValidatorException and return 422 + } + UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } - existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); return studyByName; } @@ -1090,7 +1191,59 @@ private Map> initializeUniqueLocati existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); return locationByName; } + private Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, program, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + private Optional> getTrialPIO(List experimentImportRows) { + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + + if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { + return Optional.empty(); + } + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + return Optional.ofNullable(this.trialByNameNoScope.get(expTitle.get())); + } + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Program program, Map> obsVarDatasetByName) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByName.put(existingList.getListName(), + new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceID()))); + } private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); @@ -1205,7 +1358,6 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr } private void processAndCacheTrial(BrAPITrial existingTrial, Program program, String trialRefSource, Map> trialByNameNoScope) { - existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); //get TrialId from existingTrial BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource) @@ -1213,7 +1365,7 @@ private void processAndCacheTrial(BrAPITrial existingTrial, Program program, Str UUID experimentId = UUID.fromString(experimentIDRef.getReferenceID()); trialByNameNoScope.put( - existingTrial.getTrialName(), + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java index 18ba5f851..41f432f72 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorData.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.services.processors; +import io.reactivex.functions.Function; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -45,5 +46,17 @@ static List getNewObjects(Map> objectsByName .map(preview -> preview.getBrAPIObject()) .collect(Collectors.toList()); } - + static Map getMutationsByObjectId(Map> objectsByName, Function dbIdFilter) { + return objectsByName.entrySet().stream() + .filter(entry -> ImportObjectState.MUTATED == entry.getValue().getState()) + .collect(Collectors + .toMap(entry -> { + try { + return dbIdFilter.apply(entry.getValue().getBrAPIObject()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }, + entry -> entry.getValue().getBrAPIObject())); + } } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 3abb9dcc8..9d15d225e 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -19,10 +19,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import io.reactivex.functions.Consumer; -import io.reactivex.functions.Function; -import io.reactivex.functions.Function3; -import io.reactivex.functions.Function4; +import io.reactivex.functions.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -341,8 +338,31 @@ public List post(List brapiObjects, } } + public T put(String dbId, + T brapiObject, + BiFunction putMethod) throws ApiException { + try { + ApiResponse response = putMethod.apply(dbId, brapiObject); + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } + BrAPIResponse body = (BrAPIResponse) response.getBody(); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } + return (T) body.getResult(); + + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e)); + throw e; + } catch (Exception e) { + throw new InternalServerException(e.toString(), e); + } + } + public List post(List brapiObjects, Function, ApiResponse> postMethod) throws ApiException { return post(brapiObjects, null, postMethod, null); } + } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 3f8df5c6c..6cd02e483 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -164,11 +164,13 @@ public void setup() { - new env, existing location - new experiment, missing required cols (check all required cols) - new env, missing required cols (check all req cols) + - new exp/env with observation variables (new dataset) - new exp/env with observations - new exp/env with invalid observations - existing env with missing OU ID - existing env with new observations - existing env that already has obs + - existing env that already has observation variables (existing dataset) */ @Test @@ -386,6 +388,42 @@ public void verifyMissingDataThrowsError() { uploadAndVerifyFailure(program, writeDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR); } + @Test + @SneakyThrows + public void importNewExpWithObsVar() { + List traits = createTraits(1); + Program program = createProgram("New Exp with Observations Vars", "EXPVRR", "EXPVRR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newExp, program, traits); + } + @Test @SneakyThrows public void importNewExpWithObs() { @@ -483,6 +521,70 @@ public void verifyFailureNewOuExistingEnv() { assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Units are missing Observation Unit Id.")); } + @Test + @SneakyThrows + public void importNewObsVarExisingOu() { + List traits = createTraits(2); + Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); + + importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObsVar.put(traits.get(1).getObservationVariableName(), null); + + JsonObject result = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newObsVar, program, traits); + } + @Test @SneakyThrows public void importNewObsExisingOu() { @@ -707,7 +809,12 @@ private Map assertRowSaved(Map expected, Program List observations = null; if(traits != null) { observations = observationDAO.getObservationsByStudyName(List.of(study.getStudyName()), program); - assertFalse(observations.isEmpty()); + if (expected.get(traits.get(0).getObservationVariableName()) == null) { + assertTrue(observations.isEmpty()); + } else { + assertFalse(observations.isEmpty()); + } + ret.put("observations", observations); } @@ -764,10 +871,15 @@ private Map assertRowSaved(Map expected, Program List actualVariableObservation = new ArrayList<>(); observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); for(Trait trait : traits) { - expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + if (expected.get(trait.getObservationVariableName()) != null) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + } + if (actualVariableObservation.isEmpty()) { + assertTrue(expectedVariableObservation.isEmpty()); + } else { + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); } - - assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); } return ret; diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 7a9373db6..d9ae3eecf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -40,6 +40,7 @@ import java.io.File; import java.util.Map; +import java.util.regex.Pattern; import static io.micronaut.http.HttpRequest.*; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -51,6 +52,8 @@ */ public class ImportTestUtils { + Pattern UUID_REGEX = Pattern.compile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"); + public Program insertAndFetchTestProgram(ProgramRequest programRequest, RxHttpClient client, Gson gson) { Flowable> call = client.exchange(