diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 58f0653bc..c4dd54233 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -294,15 +294,15 @@ private String processBreedbasePedigree(String pedigree) { public List createBrAPIGermplasm(List postBrAPIGermplasmList, UUID programId, ImportUpload upload) { GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); var program = programDAO.fetchOneById(programId); - Callable> postFunction = null; try { if (!postBrAPIGermplasmList.isEmpty()) { - postFunction = () -> { + Callable> postFunction = () -> { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); return processGermplasmForDisplay(postResponse, program.getKey()); }; + return programGermplasmCache.post(programId, postFunction); } - return programGermplasmCache.post(programId, postFunction); + return new ArrayList<>(); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } @@ -311,15 +311,15 @@ public List createBrAPIGermplasm(List postBrAPIG public List updateBrAPIGermplasm(List putBrAPIGermplasmList, UUID programId, ImportUpload upload) { GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); var program = programDAO.fetchOneById(programId); - Callable> postFunction = null; try { if (!putBrAPIGermplasmList.isEmpty()) { - postFunction = () -> { + Callable> postFunction = () -> { List putResponse = putGermplasm(putBrAPIGermplasmList, api); return processGermplasmForDisplay(putResponse, program.getKey()); }; + return programGermplasmCache.post(programId, postFunction); } - return programGermplasmCache.post(programId, postFunction); + return new ArrayList<>(); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index b52aeff22..6eb4c2761 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -16,19 +16,28 @@ */ package org.breedinginsight.brapi.v2.dao; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationsApi; import org.brapi.v2.model.BrAPIAcceptedSearchResponse; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationSingleResponse; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; @@ -38,6 +47,8 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -47,17 +58,115 @@ public class BrAPIObservationDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private BrAPIObservationUnitDAO observationUnitDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final String referenceSource; + private boolean runScheduledTasks; + private final ProgramCache programObservationCache; @Inject - public BrAPIObservationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIObservationDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIObservationUnitDAO observationUnitDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource, + @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks, + ProgramCacheProvider programCacheProvider) { this.programDAO = programDAO; this.importDAO = importDAO; + this.observationUnitDAO = observationUnitDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.referenceSource = referenceSource; + this.runScheduledTasks = runScheduledTasks; + this.programObservationCache = programCacheProvider.getProgramCache(this::fetchProgramObservations, BrAPIObservation.class); } + @Scheduled(initialDelay = "3s") + public void setup() { + if(!runScheduledTasks) { + return; + } + // Populate the observation cache for all programs on startup. + log.debug("populating observation cache"); + List programs = programDAO.getActive(); + if (programs != null) { + programObservationCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + /** + * Fetch formatted observations for this program. + */ + private Map fetchProgramObservations(UUID programId) throws ApiException { + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); + // Get the program. + List programs = programDAO.get(programId); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given id"); + } + Program program = programs.get(0); + + // Set query params and make call. + BrAPIObservationSearchRequest observationSearch = new BrAPIObservationSearchRequest(); + observationSearch.externalReferenceIds(List.of(programId.toString())); + observationSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); + return processObservationsForCache(brAPIDAOUtil.search( + api::searchObservationsPost, + api::searchObservationsSearchResultsDbIdGet, + observationSearch + ), program.getKey()); + } + + /** + * Process a list of observations for insertion into the cache. + */ + private Map processObservationsForCache(List programObservations, String programKey) { + // Process programObservations in place (strip program key, etc.). + processObservations(programKey, programObservations); + // Build map. + Map programObservationsMap = new HashMap<>(); + log.trace("processing observationUnits for cache: " + programObservations); + for (BrAPIObservation observation: programObservations) { + BrAPIExternalReference xref = observation + .getExternalReferences() + .stream() + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATIONS.getName()).equals(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + programObservationsMap.put(xref.getReferenceId(), observation); + } + return programObservationsMap; + } + + /** + * Process BrAPIObservations for use in DeltaBreed (e.g. strip program key). + */ + private void processObservations(String programKey, List observations) { + for (BrAPIObservation obs: observations) { + // Strip program key from observationVariableName. + if (StringUtils.isNotBlank(obs.getObservationVariableName())) { + obs.setObservationVariableName(Utilities.removeProgramKey(obs.getObservationVariableName(), programKey)); + } + // Strip program key and unknown info from germplasmName and observationUnitName. + if (StringUtils.isNotBlank(obs.getGermplasmName())) { + obs.setGermplasmName(Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getGermplasmName(), programKey)); + } + if (StringUtils.isNotBlank(obs.getObservationUnitName())) { + obs.setObservationUnitName(Utilities.removeProgramKeyAndUnknownAdditionalData(obs.getObservationUnitName(), programKey)); + } + } + } + + /** + * Get all observations for a program from the cache. + */ + private Map getProgramObservations(UUID programId) throws ApiException { + return programObservationCache.get(programId); + } + + // Note: not using cache, because unique studyName (with "[ProgramKey-ExtraInfo]") is not stored directly on Observation. public List getObservationsByStudyName(List studyNames, Program program) throws ApiException { if(studyNames.isEmpty()) { return Collections.emptyList(); @@ -78,33 +187,22 @@ public List getObservationsByTrialDbId(List trialDbIds if(trialDbIds.isEmpty()) { return Collections.emptyList(); } - - BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); - observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - observationSearchRequest.setTrialDbIds(new ArrayList<>(trialDbIds)); - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); - return brAPIDAOUtil.search( - api::searchObservationsPost, - (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), - observationSearchRequest - ); + // First, get all ObservationUnits for the given trialDbIds. + List observationUnitDbIds = observationUnitDAO.getObservationUnitsForTrialDbIds(program.getId(), trialDbIds) + .stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); + // Finally, return all Observations for those ObservationUnits (Observations are linked to Trial through ObservationUnits). + return getProgramObservations(program.getId()).values().stream() + .filter(o -> observationUnitDbIds.contains(o.getObservationUnitDbId())) + .collect(Collectors.toList()); } public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { if(ouDbIds.isEmpty() || variableDbIds.isEmpty()) { return Collections.emptyList(); } - - BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); - observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - observationSearchRequest.setObservationUnitDbIds(new ArrayList<>(ouDbIds)); - observationSearchRequest.setObservationVariableDbIds(new ArrayList<>(variableDbIds)); - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); - return brAPIDAOUtil.search( - api::searchObservationsPost, - (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), - observationSearchRequest - ); + return getProgramObservations(program.getId()).values().stream() + .filter(o -> ouDbIds.contains(o.getObservationUnitDbId()) && variableDbIds.contains(o.getObservationVariableDbId())) + .collect(Collectors.toList()); } @NotNull @@ -115,28 +213,47 @@ private ApiResponse, Optional createBrAPIObservations(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); - return brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); + var program = programDAO.fetchOneById(programId); + try { + if (!brAPIObservationList.isEmpty()) { + Callable> postFunction = () -> { + List postResponse = brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); + return processObservationsForCache(postResponse, program.getKey()); + }; + return programObservationCache.post(programId, postFunction); + } + return new ArrayList<>(); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } } public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation observation, UUID programId) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); - ApiResponse response; - BrAPIObservation updatedObservation = null; + var program = programDAO.fetchOneById(programId); try { - response = api.observationsObservationDbIdPut(dbId, observation); - if (response != null) { - BrAPIObservationSingleResponse body = response.getBody(); - if (body == null) { - throw new ApiException("Response is missing body", 0, response.getHeaders(), null); - } - updatedObservation = body.getResult(); - if (updatedObservation == null) { - throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); - } - } + Callable> postFunction = () -> { + ApiResponse response = api.observationsObservationDbIdPut(dbId, observation); + if (response == null) + { + throw new ApiException("Response is null", 0, null, null); + } + BrAPIObservationSingleResponse body = response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); + } + BrAPIObservation updatedObservation = body.getResult(); + if (updatedObservation == null) { + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); + } + return processObservationsForCache(List.of(updatedObservation), program.getKey()); + }; + return programObservationCache.post(programId, postFunction).get(0); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } - return updatedObservation; } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 40833dc3e..7d8fd9651 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -23,10 +23,13 @@ import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationTreatment; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -38,6 +41,8 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -50,9 +55,11 @@ import javax.validation.constraints.NotNull; import java.lang.reflect.Type; import java.util.*; +import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +@Slf4j @Singleton public class BrAPIObservationUnitDAO { private final ProgramDAO programDAO; @@ -63,10 +70,13 @@ public class BrAPIObservationUnitDAO { private final BrAPIGermplasmService germplasmService; private final String referenceSource; + private final boolean runScheduledTasks; private final Gson gson = new JSON().getGson(); private final Type treatmentlistType = new TypeToken>(){}.getType(); + private final ProgramCache programObservationUnitCache; + @Inject public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, @@ -74,26 +84,89 @@ public BrAPIObservationUnitDAO(ProgramDAO programDAO, BrAPIEndpointProvider brAPIEndpointProvider, BrAPIGermplasmService germplasmService, ProgramService programService, - @Property(name = "brapi.server.reference-source") String referenceSource) { + @Property(name = "brapi.server.reference-source") String referenceSource, + @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks, + ProgramCacheProvider programCacheProvider) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; this.referenceSource = referenceSource; + this.runScheduledTasks = runScheduledTasks; this.programService = programService; this.germplasmService = germplasmService; + this.programObservationUnitCache = programCacheProvider.getProgramCache(this::fetchProgramObservationUnits, BrAPIObservationUnit.class); + } + + @Scheduled(initialDelay = "3s") + public void setup() { + if(!runScheduledTasks) { + return; + } + // Populate observation unit cache for all programs on startup. + log.debug("populating observation unit cache"); + List programs = programDAO.getActive(); + if(programs != null) { + programObservationUnitCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + /** + * Fetch formatted observation units for this program. + */ + private Map fetchProgramObservationUnits(UUID programId) throws ApiException { + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); + // Get the program. + List programs = programDAO.get(programId); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given id"); + } + Program program = programs.get(0); + + // Set query params and make call. + BrAPIObservationUnitSearchRequest observationUnitSearch = new BrAPIObservationUnitSearchRequest(); + observationUnitSearch.externalReferenceIds(List.of(programId.toString())); + observationUnitSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); + return processObservationUnitsForCache(brAPIDAOUtil.search( + api::searchObservationunitsPost, + api::searchObservationunitsSearchResultsDbIdGet, + observationUnitSearch + ), program, true); + } + + /** + * Process a list of observation units for insertion into the cache. + */ + private Map processObservationUnitsForCache(List programObservationUnits, Program program, boolean withGID) throws ApiException { + // Process programObservationUnits in place (strip program key, etc.). + processObservationUnits(program, programObservationUnits, withGID); + // Build map. + Map programObservationUnitsMap = new HashMap<>(); + for (BrAPIObservationUnit observationUnit: programObservationUnits) { + BrAPIExternalReference xref = observationUnit + .getExternalReferences() + .stream() + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()).equals(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + programObservationUnitsMap.put(xref.getReferenceId(), observationUnit); + } + return programObservationUnitsMap; + } + + /** + * Get all observation units for a program from the cache. + */ + private Map getProgramObservationUnits(UUID programId) throws ApiException { + return programObservationUnitCache.get(programId); } public List getObservationUnitByName(List observationUnitNames, Program program) throws ApiException { if(observationUnitNames.isEmpty()) { return Collections.emptyList(); } - - BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); - observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - observationUnitSearchRequest.observationUnitNames(observationUnitNames); - - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, false); + return getProgramObservationUnits(program.getId()).values().stream() + .filter(ou -> observationUnitNames.contains(ou.getObservationUnitName())) + .collect(Collectors.toList()); } /** @@ -102,20 +175,16 @@ public List getObservationUnitByName(List observat public List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId, ImportUpload upload) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); - preprocessObservationUnits(brAPIObservationUnitList); - List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); - processObservationUnits(program, ous, false); - return ous; - } - - public BrAPIObservationUnit updateBrAPIObservationUnit(String ouDbId, BrAPIObservationUnit ou, UUID programId) { - ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); - BrAPIObservationUnit updatedOu = null; try { - if (ou != null && !ouDbId.isBlank()) { - updatedOu = brAPIDAOUtil.put(ouDbId, ou, api::observationunitsObservationUnitDbIdPut); + if (!brAPIObservationUnitList.isEmpty()) { + Callable> postFunction = () -> { + preprocessObservationUnits(brAPIObservationUnitList); + List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); + return processObservationUnitsForCache(ous, program, false); + }; + return programObservationUnitCache.post(programId, postFunction); } - return updatedOu; + return new ArrayList<>(); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } @@ -125,49 +194,44 @@ public List getObservationUnitsById(Collection obs if(observationUnitExternalIds.isEmpty()) { return Collections.emptyList(); } - - BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); - observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() - .getProgramDbId())); - observationUnitSearchRequest.externalReferenceIDs(new ArrayList<>(observationUnitExternalIds)); - observationUnitSearchRequest.externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))); - - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, false); + return getProgramObservationUnits(program.getId()).entrySet().stream() + .filter(entry -> observationUnitExternalIds.contains(entry.getKey())) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); } public List getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException { - BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); - observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() - .getProgramDbId())); - observationUnitSearchRequest.studyDbIds(List.of(studyDbId)); - - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, false); + return getProgramObservationUnits(program.getId()).values().stream() + .filter(ou -> ou.getStudyDbId().equals(studyDbId)) + .collect(Collectors.toList()); } - public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId) throws ApiException, DoesNotExistException { - return getObservationUnitsForTrialDbId(programId, trialDbId, false); + public List getObservationUnitsForTrialDbIds(@NotNull UUID programId, List trialDbIds) throws ApiException { + if (trialDbIds.isEmpty()) { + return Collections.emptyList(); + } + return getProgramObservationUnits(programId).values().stream() + .filter(ou -> trialDbIds.contains(ou.getTrialDbId())) + .collect(Collectors.toList()); } - public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId, boolean withGID) throws ApiException, DoesNotExistException { - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - - BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); - observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() - .getProgramDbId())); - observationUnitSearchRequest.trialDbIds(List.of(trialDbId)); - - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, withGID); + public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId) throws ApiException { + return getProgramObservationUnits(programId).values().stream() + .filter(ou -> ou.getTrialDbId().equals(trialDbId)) + .collect(Collectors.toList()); } public List getObservationUnitsForDataset(@NotNull String datasetId, @NotNull Program program) throws ApiException { String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); - BrAPIObservationUnitSearchRequest ouSearchRequest = new BrAPIObservationUnitSearchRequest(); - ouSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - ouSearchRequest.externalReferenceSources(List.of(datasetReferenceSource)); - ouSearchRequest.externalReferenceIDs(List.of(datasetId)); - return searchObservationUnitsAndProcess(ouSearchRequest, program, true); + return getProgramObservationUnits(program.getId()).values().stream() + .filter(ou -> { + Optional exRef = Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource); + return exRef.map(brAPIExternalReference -> brAPIExternalReference.getReferenceId().equals(datasetId)).orElse(false); + }) + .collect(Collectors.toList()); } + // Note: does not use cache, impractical to implement all search parameters client-side. public List getObservationUnits(Program program, Optional observationUnitId, Optional observationUnitName, @@ -227,15 +291,15 @@ public List getObservationUnits(Program program, //xref search does an OR, so we need to convert the searching for ouId/expId/envId to be an AND boolean matches = observationUnitId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) .get() - .getReferenceID())) + .getReferenceId())) .orElse(true); matches = matches && experimentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) .get() - .getReferenceID())) + .getReferenceId())) .orElse(true); matches = matches && environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) .get() - .getReferenceID())) + .getReferenceId())) .orElse(true); //adding filter for germplasmDbId because we can't easily search that in the stored data object @@ -303,7 +367,7 @@ private void processObservationUnits(Program program, List ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_UUID, Utilities.getExternalReference(germplasm.getExternalReferences(), referenceSource) .orElseThrow(() -> new IllegalStateException("Germplasm UUID not found")) - .getReferenceID()); + .getReferenceId()); } } ou.setObservationUnitName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java index 2901baa4e..870b0579b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -189,7 +189,7 @@ private Map environmentById(List studies) { BrAPIExternalReference xref = environment .getExternalReferences() .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES).equalsIgnoreCase(reference.getReferenceSource())) + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES.getName()).equals(reference.getReferenceSource())) .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); environmentById.put(xref.getReferenceID(), environment); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java index a60008dcd..a1670bf3c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java @@ -124,7 +124,7 @@ private Map experimentById(List trials) { BrAPIExternalReference xref = experiment .getExternalReferences() .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS).equalsIgnoreCase(reference.getReferenceSource())) + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS.getName()).equals(reference.getReferenceSource())) .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); experimentById.put(xref.getReferenceID(), experiment); } diff --git a/src/main/java/org/breedinginsight/daos/ObservationDAO.java b/src/main/java/org/breedinginsight/daos/ObservationDAO.java index 9ecfb0588..0802c923f 100644 --- a/src/main/java/org/breedinginsight/daos/ObservationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ObservationDAO.java @@ -18,19 +18,15 @@ import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.BrAPIClient; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.phenotype.ObservationQueryParams; import org.brapi.client.v2.modules.phenotype.ObservationsApi; -import org.brapi.v2.model.BrAPIAcceptedSearchResponse; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; -import org.breedinginsight.services.brapi.BrAPIClientType; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; -import org.breedinginsight.services.brapi.BrAPIProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; @@ -40,8 +36,6 @@ import java.util.List; import java.util.UUID; -import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; - @Singleton @Slf4j diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 87fec8b46..3e084752d 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -192,12 +192,17 @@ public static Table parseTableFromCsv(InputStream inputStream) throws ParsingExc } public static Table parseTableFromJson(String jsonString) throws ParsingException { + try { return Table.read() .usingOptions( JsonReadOptions .builderFromString(jsonString) .columnTypesToDetect(List.of(ColumnType.STRING)) ); + } catch (Exception e) { + log.debug(e.getMessage()); + throw new ParsingException(ParsingExceptionType.ERROR_READING_FILE); + } } public static Table removeNullRows(Table table) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java index b7a2ba564..c4c214682 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java @@ -2,11 +2,13 @@ import com.google.gson.Gson; import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import lombok.SneakyThrows; import org.brapi.client.v2.JSON; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationTreatment; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.BrAPITest; @@ -17,17 +19,20 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; +import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; import javax.inject.Inject; import java.util.List; +import java.util.UUID; import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -65,6 +70,9 @@ public class BrAPIObservationUnitDAOTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") RxHttpClient biClient; + @Property(name = "brapi.server.reference-source") + String referenceSource; + @BeforeAll @SneakyThrows public void setup() { @@ -117,6 +125,11 @@ public void testCreateObservationUnitAdditionalInfoSingleTreatmentFactor() { ou1.setObservationUnitName("test1"); ou1.putAdditionalInfoItem(BrAPIAdditionalInfoFields.TREATMENTS, List.of(testTreatment)); ou1.setProgramDbId(validProgram.getBrapiProgram().getProgramDbId()); + // Set xref. + BrAPIExternalReference xref = new BrAPIExternalReference(); + xref.setReferenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)); + xref.setReferenceId(UUID.randomUUID().toString()); + ou1.setExternalReferences(List.of(xref)); List ous = List.of(ou1); List createdOus = obsUnitDAO.createBrAPIObservationUnits(ous, validProgram.getId(), upload);