From e103938394ec76c88dc4b0890812c3bedeec07c2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:03:42 -0400 Subject: [PATCH 01/16] cache exoeriments at startup and use for create and update --- .../brapps/importer/daos/BrAPITrialDAO.java | 105 +++++++++++++++++- 1 file changed, 99 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 0a9180936..14682cf1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -16,14 +16,20 @@ */ package org.breedinginsight.brapps.importer.daos; +import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -31,26 +37,30 @@ import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; +import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; +import java.util.concurrent.Callable; import java.util.stream.Collectors; +@Slf4j +@Context @Singleton public class BrAPITrialDAO { @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; - + private final ProgramCache programExperimentCache; private final ProgramDAO programDAO; private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final ProgramService programService; private final BrAPIEndpointProvider brAPIEndpointProvider; - private final String referenceSource; @Inject - public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPITrialDAO(ProgramCacheProvider programCacheProvider, ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIEndpointProvider brAPIEndpointProvider) { + this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiment, BrAPITrial.class); this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; @@ -59,6 +69,54 @@ public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil br this.brAPIEndpointProvider = brAPIEndpointProvider; } + @PostConstruct + public void setup() { + // Populate the experiment cache for all programs on startup + log.debug("populating experiment cache"); + List programs = programDAO.getActive(); + if (programs != null) { + programExperimentCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + private Map fetchProgramExperiment(UUID programId) throws ApiException { + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + + // Get the program + List programs = programDAO.get(programId); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given key"); + } + Program program = programs.get(0); + + // Get the program experiments + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.externalReferenceIDs(List.of(programId.toString())); + trialSearch.externalReferenceSources( + List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + ); + List programExperiments = brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + + return experimentById(programExperiments); + } + + private Map experimentById(List trials) { + Map experimentById = new HashMap<>(); + for (BrAPITrial experiment: trials) { + BrAPIExternalReference xref = experiment + .getExternalReferences() + .stream() + .filter(reference -> referenceSource.equals(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + experimentById.put(xref.getReferenceID(), experiment); + } + return experimentById; + } + public List getTrialsByName(List trialNames, Program program) throws ApiException { if(trialNames.isEmpty()) { return Collections.emptyList(); @@ -88,20 +146,51 @@ private List getTrialsByExRef(String referenceSource, String referen ); } - public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) + throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - return brAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); + Callable> postCallback = null; + try { + if (!brAPITrialList.isEmpty()) { + postCallback = () -> { + List postedTrials = brAPIDAOUtil + .post(brAPITrialList, upload, api::trialsPost, importDAO::update); + return experimentById(postedTrials); + }; + } + return programExperimentCache.post(programId, postCallback); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } } public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - return brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); + Callable> putCallback = null; + try { + if (trial != null) { + putCallback = () -> { + BrAPITrial updatedTrial = brAPIDAOUtil + .put(trialDbId, trial, api::trialsTrialDbIdPut); + return experimentById(List.of(updatedTrial)); + }; + } + List cachedUpdates = programExperimentCache.post(programId, putCallback); + if (cachedUpdates.isEmpty()) { + throw new Exception(); + } + return cachedUpdates.get(0); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } } + /** * Fetch formatted trials/experiments for this program * @param programId * @return List * @throws ApiException */ + /* public List getTrials(UUID programId) throws ApiException, DoesNotExistException { BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); //TODO check external references filter works once implemented in BI-1552 @@ -116,6 +205,10 @@ public List getTrials(UUID programId) throws ApiException, DoesNotEx api::searchTrialsPost, api::searchTrialsSearchResultsDbIdGet, trialSearch), program.getKey(), programId, true); + }*/ + + public List getTrials(UUID programId) throws ApiException { + return new ArrayList<>(programExperimentCache.get(programId).values()); } //Removes program key from trial name and adds dataset information From a9f783f39084a0da5d19f78250878bd5f546de26 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:59:15 -0400 Subject: [PATCH 02/16] add caching to getTrialByDbId --- .../brapps/importer/daos/BrAPITrialDAO.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 14682cf1f..02c9b57e7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -229,13 +229,34 @@ private List processExperimentsForDisplay(List trials, S return displayExperiments; } + public Optional getTrialById(UUID programId, UUID trialDbId) throws ApiException, DoesNotExistException { + Program program = programService + .getById(programId) + .orElseThrow(() -> new DoesNotExistException("Program id does not exist")); + Map cache = programExperimentCache.get(program.getId()); + BrAPITrial trial = null; + if (cache != null) { + List trials = cache + .values() + .stream() + .filter(t -> t.getTrialDbId().equals(trialDbId)) + .collect(Collectors.toList()); + if (trials.isEmpty()) { + throw new DoesNotExistException("trial does not exist for given dbId"); + } + trial = trials.get(0); + } + + return Optional.ofNullable(trial); + } + /* public Optional getTrialById(UUID programId, UUID trialDbId) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); String refSoure = Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS); List trials = getTrialsByExRef(refSoure, trialDbId.toString(), program); return Utilities.getSingleOptional(trials); - } + }*/ public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { List trials = getTrialsByDbIds(List.of(trialDbId), program); From 65f96a40c3d12c7739a914f1177ce5632915bd59 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 12 Jun 2023 12:08:02 -0400 Subject: [PATCH 03/16] add caching to getTrialByNames --- .../brapps/importer/daos/BrAPITrialDAO.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 02c9b57e7..cb9216f6e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -117,6 +117,7 @@ private Map experimentById(List trials) { return experimentById; } + /* public List getTrialsByName(List trialNames, Program program) throws ApiException { if(trialNames.isEmpty()) { return Collections.emptyList(); @@ -131,6 +132,19 @@ public List getTrialsByName(List trialNames, Program program api::searchTrialsSearchResultsDbIdGet, trialSearch ); + }*/ + public List getTrialsByName(List trialNames, Program program) throws ApiException { + Map cache = programExperimentCache.get(program.getId()); + List trials = new ArrayList<>(); + if (cache != null) { + trials.addAll(cache + .values() + .stream() + .filter(t -> trialNames.contains(t.getTrialName())) + .collect(Collectors.toList())); + } + + return trials; } private List getTrialsByExRef(String referenceSource, String referenceId, Program program) throws ApiException { From cc768c21174923b61e8a7ffb24006543189c6569 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 12 Jun 2023 12:12:52 -0400 Subject: [PATCH 04/16] add caching to getTrialByDbIds --- .../brapps/importer/daos/BrAPITrialDAO.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index cb9216f6e..92fc76745 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -276,40 +276,37 @@ public Optional getTrialByDbId(String trialDbId, Program program) th List trials = getTrialsByDbIds(List.of(trialDbId), program); return Utilities.getSingleOptional(trials); } + public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { - if(trialDbIds.isEmpty()) { - return Collections.emptyList(); + Map cache = programExperimentCache.get(program.getId()); + List trials = new ArrayList<>(); + if (cache != null) { + trials.addAll(cache + .values() + .stream() + .filter(t -> trialDbIds.contains(t.getTrialDbId())) + .collect(Collectors.toList())); } - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - trialSearch.trialDbIds(new ArrayList<>(trialDbIds)); - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - return brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); + return trials; } - public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { - if(experimentIds.isEmpty()) { + /* + public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { + if(trialDbIds.isEmpty()) { return Collections.emptyList(); } BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - //trialSearch.trialDbIds(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); - trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); - trialSearch.externalReferenceIDs(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + trialSearch.trialDbIds(new ArrayList<>(trialDbIds)); TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); return brAPIDAOUtil.search( api::searchTrialsPost, api::searchTrialsSearchResultsDbIdGet, trialSearch ); - } - + }*/ } From d62d63c14241d245237306acb53d93ba38a5e515 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 12 Jun 2023 14:12:52 -0400 Subject: [PATCH 05/16] fix xref check --- .../breedinginsight/brapps/importer/daos/BrAPITrialDAO.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 92fc76745..52477a1ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -20,6 +20,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.core.TrialsApi; import org.brapi.v2.model.BrAPIExternalReference; @@ -110,7 +111,7 @@ private Map experimentById(List trials) { BrAPIExternalReference xref = experiment .getExternalReferences() .stream() - .filter(reference -> referenceSource.equals(reference.getReferenceSource())) + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS).equalsIgnoreCase(reference.getReferenceSource())) .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); experimentById.put(xref.getReferenceID(), experiment); } From 4e0e27121565fc729ac122e159e0fb9ce0cc4b3f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 12 Jun 2023 16:28:40 -0400 Subject: [PATCH 06/16] add cache to remaining methods in trialDAO --- .../brapps/importer/daos/BrAPITrialDAO.java | 147 ++++++------------ 1 file changed, 44 insertions(+), 103 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 52477a1ac..b7e1e4508 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -118,22 +118,6 @@ private Map experimentById(List trials) { return experimentById; } - /* - public List getTrialsByName(List trialNames, Program program) throws ApiException { - if(trialNames.isEmpty()) { - return Collections.emptyList(); - } - - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - trialSearch.trialNames(trialNames); - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - return brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); - }*/ public List getTrialsByName(List trialNames, Program program) throws ApiException { Map cache = programExperimentCache.get(program.getId()); List trials = new ArrayList<>(); @@ -149,85 +133,78 @@ public List getTrialsByName(List trialNames, Program program } private List getTrialsByExRef(String referenceSource, String referenceId, Program program) throws ApiException { - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - trialSearch.externalReferenceSources(List.of(referenceSource)); - trialSearch.externalReferenceIDs(List.of(referenceId)); - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - return brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); + Map cache = programExperimentCache.get(program.getId()); + List trials = new ArrayList<>(); + if (cache != null) { + trials.addAll(cache + .values() + .stream() + .filter(t -> { + BrAPIExternalReference xref = t + .getExternalReferences() + .stream() + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS) + .equalsIgnoreCase(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI trial external reference found")); + return referenceId.equals(xref.getReferenceID()); + }) + .collect(Collectors.toList())); + } + + return trials; } public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - Callable> postCallback = null; + List createdTrials = new ArrayList<>(); try { if (!brAPITrialList.isEmpty()) { - postCallback = () -> { + Callable> postCallback = () -> { List postedTrials = brAPIDAOUtil .post(brAPITrialList, upload, api::trialsPost, importDAO::update); return experimentById(postedTrials); }; + createdTrials.addAll(programExperimentCache.post(programId, postCallback)); } - return programExperimentCache.post(programId, postCallback); + + return createdTrials; } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } } public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - Callable> putCallback = null; + BrAPITrial updatedTrial = null; try { - if (trial != null) { - putCallback = () -> { - BrAPITrial updatedTrial = brAPIDAOUtil - .put(trialDbId, trial, api::trialsTrialDbIdPut); - return experimentById(List.of(updatedTrial)); + if (trial != null && !trialDbId.isBlank()) { + Callable> putCallback = () -> { + BrAPITrial putTrial = brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); + return experimentById(List.of(putTrial)); }; + List cachedUpdates = programExperimentCache.post(programId, putCallback); + if (cachedUpdates.isEmpty()) { + throw new Exception(); + } + updatedTrial = cachedUpdates.get(0); } - List cachedUpdates = programExperimentCache.post(programId, putCallback); - if (cachedUpdates.isEmpty()) { - throw new Exception(); - } - return cachedUpdates.get(0); + + return updatedTrial; } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } } - /** - * Fetch formatted trials/experiments for this program - * @param programId - * @return List - * @throws ApiException - */ - /* - public List getTrials(UUID programId) throws ApiException, DoesNotExistException { - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - //TODO check external references filter works once implemented in BI-1552 - trialSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - trialSearch.externalReferenceIDs(List.of(programId.toString())); - - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - return processExperimentsForDisplay(brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch), program.getKey(), programId, true); - }*/ - public List getTrials(UUID programId) throws ApiException { return new ArrayList<>(programExperimentCache.get(programId).values()); } //Removes program key from trial name and adds dataset information - private List processExperimentsForDisplay(List trials, String programKey, UUID programId, boolean metadata) throws ApiException { + private List processExperimentsForDisplay( + List trials, + String programKey, + UUID programId, + boolean metadata) throws ApiException { List displayExperiments = new ArrayList<>(); for (BrAPITrial trial: trials) { trial.setTrialName(Utilities.removeProgramKey(trial.getTrialName(), programKey, "")); @@ -244,34 +221,15 @@ private List processExperimentsForDisplay(List trials, S return displayExperiments; } - public Optional getTrialById(UUID programId, UUID trialDbId) throws ApiException, DoesNotExistException { - Program program = programService - .getById(programId) - .orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - Map cache = programExperimentCache.get(program.getId()); + public Optional getTrialById(UUID programId, UUID trialId) throws ApiException, DoesNotExistException { + Map cache = programExperimentCache.get(programId); BrAPITrial trial = null; if (cache != null) { - List trials = cache - .values() - .stream() - .filter(t -> t.getTrialDbId().equals(trialDbId)) - .collect(Collectors.toList()); - if (trials.isEmpty()) { - throw new DoesNotExistException("trial does not exist for given dbId"); - } - trial = trials.get(0); + trial = cache.get(trialId); } return Optional.ofNullable(trial); } - /* - public Optional getTrialById(UUID programId, UUID trialDbId) throws ApiException, DoesNotExistException { - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - String refSoure = Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS); - List trials = getTrialsByExRef(refSoure, trialDbId.toString(), program); - - return Utilities.getSingleOptional(trials); - }*/ public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { List trials = getTrialsByDbIds(List.of(trialDbId), program); @@ -291,23 +249,6 @@ public List getTrialsByDbIds(Collection trialDbIds, Program return trials; } - - /* - public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { - if(trialDbIds.isEmpty()) { - return Collections.emptyList(); - } - - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - trialSearch.trialDbIds(new ArrayList<>(trialDbIds)); - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - return brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); - }*/ } From d4f20894453861dbe9eccd88ab7065f990bd77bb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 13 Jun 2023 13:20:28 -0400 Subject: [PATCH 07/16] apply fix --- .../org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index b7e1e4508..e2fac6ea7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -19,6 +19,7 @@ import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -70,6 +71,7 @@ public BrAPITrialDAO(ProgramCacheProvider programCacheProvider, ProgramDAO progr this.brAPIEndpointProvider = brAPIEndpointProvider; } + @PostConstruct public void setup() { // Populate the experiment cache for all programs on startup From e98fca2691aa210dc9122641a516ceef83299271 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 13 Jun 2023 14:25:01 -0400 Subject: [PATCH 08/16] remove key from trial name --- .../brapps/importer/daos/BrAPITrialDAO.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index e2fac6ea7..62c1fd33e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -104,7 +104,7 @@ private Map fetchProgramExperiment(UUID programId) throws Ap trialSearch ); - return experimentById(programExperiments); + return experimentById(processExperimentsForDisplay(programExperiments, program.getKey())); } private Map experimentById(List trials) { @@ -201,23 +201,13 @@ public List getTrials(UUID programId) throws ApiException { return new ArrayList<>(programExperimentCache.get(programId).values()); } - //Removes program key from trial name and adds dataset information + //Removes program key from trial name private List processExperimentsForDisplay( List trials, - String programKey, - UUID programId, - boolean metadata) throws ApiException { + String programKey) throws ApiException { List displayExperiments = new ArrayList<>(); for (BrAPITrial trial: trials) { trial.setTrialName(Utilities.removeProgramKey(trial.getTrialName(), programKey, "")); - List datasets = new ArrayList<>(); - - if (metadata) { - //todo presumably BI-1193 replace dummy value with list of datasets once datasets implemented - datasets.add("Observation Dataset"); - trial.putAdditionalInfoItem("datasets", datasets); - } - displayExperiments.add(trial); } return displayExperiments; @@ -227,7 +217,7 @@ public Optional getTrialById(UUID programId, UUID trialId) throws Ap Map cache = programExperimentCache.get(programId); BrAPITrial trial = null; if (cache != null) { - trial = cache.get(trialId); + trial = cache.get(trialId.toString()); } return Optional.ofNullable(trial); From 2d2616729aa451f3561b2ba9ed6dbc7a83c1d066 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 23 Jun 2023 15:35:19 -0400 Subject: [PATCH 09/16] respond to comments --- .../breedinginsight/brapps/importer/daos/BrAPITrialDAO.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 62c1fd33e..5ec5aea87 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -62,7 +62,7 @@ public class BrAPITrialDAO { @Inject public BrAPITrialDAO(ProgramCacheProvider programCacheProvider, ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIEndpointProvider brAPIEndpointProvider) { - this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiment, BrAPITrial.class); + this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiments, BrAPITrial.class); this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; @@ -82,7 +82,7 @@ public void setup() { } } - private Map fetchProgramExperiment(UUID programId) throws ApiException { + private Map fetchProgramExperiments(UUID programId) throws ApiException { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); // Get the program @@ -124,6 +124,8 @@ public List getTrialsByName(List trialNames, Program program Map cache = programExperimentCache.get(program.getId()); List trials = new ArrayList<>(); if (cache != null) { + + // TODO: replace with more performant cache search, e.g. RediSearch trials.addAll(cache .values() .stream() From 63e01e476aa4a9f67fba6285d43f41914d469b5e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 23 Jun 2023 18:14:57 -0400 Subject: [PATCH 10/16] add back getTrialsByExperimentIds --- .../brapps/importer/daos/BrAPITrialDAO.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 5ec5aea87..46260fc96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -243,6 +243,21 @@ public List getTrialsByDbIds(Collection trialDbIds, Program return trials; } + public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { + if(experimentIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); + trialSearch.externalReferenceIDs(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); + return brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + } } From 229288083900df182980cecc45ffe16a42aadc0e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:23:24 -0400 Subject: [PATCH 11/16] eagar-load trial cache --- .../brapps/importer/daos/BrAPITrialDAO.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 46260fc96..3507e2ce3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -59,7 +59,8 @@ public class BrAPITrialDAO { private final ProgramService programService; private final BrAPIEndpointProvider brAPIEndpointProvider; private final String referenceSource; - + @Property(name = "micronaut.bi.api.run-scheduled-tasks") + private boolean runScheduledTasks; @Inject public BrAPITrialDAO(ProgramCacheProvider programCacheProvider, ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIEndpointProvider brAPIEndpointProvider) { this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiments, BrAPITrial.class); @@ -72,8 +73,12 @@ public BrAPITrialDAO(ProgramCacheProvider programCacheProvider, ProgramDAO progr } - @PostConstruct + @Scheduled(initialDelay = "2s") public void setup() { + if(!runScheduledTasks) { + return; + } + // Populate the experiment cache for all programs on startup log.debug("populating experiment cache"); List programs = programDAO.getActive(); From d32a191b6e8e4ae574e64f8da38a21943fad116c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:09:44 -0400 Subject: [PATCH 12/16] update ExperimentFileImportTest0 --- .../brapps/importer/ExperimentFileImportTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index d63a0a515..af194ee3a 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -304,7 +304,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newEnv = new HashMap<>(); newEnv.put(Columns.GERMPLASM_GID, "1"); @@ -634,7 +634,7 @@ public void importNewObsVarExisingOu() { importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); - BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); @@ -699,7 +699,7 @@ public void importNewObsExisingOu(boolean commit) { importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); - BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); @@ -766,7 +766,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { importTestUtils.uploadAndFetch(writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); - BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)newExp.get(Columns.EXP_TITLE), program.getKey())), program).get(0); + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); @@ -866,7 +866,7 @@ public void importSecondExpAfterFirstExpWithObs() { private Map assertRowSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); - List trials = brAPITrialDAO.getTrialsByName(List.of(Utilities.appendProgramKey((String)expected.get(Columns.EXP_TITLE), program.getKey())), program); + List trials = brAPITrialDAO.getTrialsByName(List.of((String)expected.get(Columns.EXP_TITLE)), program); assertFalse(trials.isEmpty()); BrAPITrial trial = trials.get(0); Optional trialIdXref = Utilities.getExternalReference(trial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); From 8b022b81ee83b34b32d32788a4aa06691b4e6f2c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:16:20 -0400 Subject: [PATCH 13/16] refactor TrialDAO as interface and pull latest image for gigwa test --- .../brapps/importer/daos/BrAPITrialDAO.java | 265 +----------------- ...gwaGenotypeServiceImplIntegrationTest.java | 25 +- 2 files changed, 37 insertions(+), 253 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 3507e2ce3..55e35f6e8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -1,268 +1,31 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.breedinginsight.brapps.importer.daos; -import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.modules.core.TrialsApi; -import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; -import org.breedinginsight.services.ProgramService; -import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.breedinginsight.utilities.Utilities; -import javax.annotation.PostConstruct; -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.stream.Collectors; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.UUID; -@Slf4j -@Context -@Singleton -public class BrAPITrialDAO { - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - private final ProgramCache programExperimentCache; - private final ProgramDAO programDAO; - private final ImportDAO importDAO; - private final BrAPIDAOUtil brAPIDAOUtil; - private final ProgramService programService; - private final BrAPIEndpointProvider brAPIEndpointProvider; - private final String referenceSource; - @Property(name = "micronaut.bi.api.run-scheduled-tasks") - private boolean runScheduledTasks; - @Inject - public BrAPITrialDAO(ProgramCacheProvider programCacheProvider, ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIEndpointProvider brAPIEndpointProvider) { - this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiments, BrAPITrial.class); - this.programDAO = programDAO; - this.importDAO = importDAO; - this.brAPIDAOUtil = brAPIDAOUtil; - this.programService = programService; - this.referenceSource = referenceSource; - this.brAPIEndpointProvider = brAPIEndpointProvider; - } +public interface BrAPITrialDAO { + List getTrialsByName(List trialNames, Program program) throws ApiException; + List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) + throws ApiException; - @Scheduled(initialDelay = "2s") - public void setup() { - if(!runScheduledTasks) { - return; - } + BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException; - // Populate the experiment cache for all programs on startup - log.debug("populating experiment cache"); - List programs = programDAO.getActive(); - if (programs != null) { - programExperimentCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } - } + List getTrials(UUID programId) throws ApiException; - private Map fetchProgramExperiments(UUID programId) throws ApiException { - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + Optional getTrialById(UUID programId, UUID trialId) throws ApiException, DoesNotExistException; - // Get the program - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given key"); - } - Program program = programs.get(0); + Optional getTrialByDbId(String trialDbId, Program program) throws ApiException; - // Get the program experiments - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.externalReferenceIDs(List.of(programId.toString())); - trialSearch.externalReferenceSources( - List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) - ); - List programExperiments = brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); + List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException; - return experimentById(processExperimentsForDisplay(programExperiments, program.getKey())); - } - - private Map experimentById(List trials) { - Map experimentById = new HashMap<>(); - for (BrAPITrial experiment: trials) { - BrAPIExternalReference xref = experiment - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS).equalsIgnoreCase(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - experimentById.put(xref.getReferenceID(), experiment); - } - return experimentById; - } - - public List getTrialsByName(List trialNames, Program program) throws ApiException { - Map cache = programExperimentCache.get(program.getId()); - List trials = new ArrayList<>(); - if (cache != null) { - - // TODO: replace with more performant cache search, e.g. RediSearch - trials.addAll(cache - .values() - .stream() - .filter(t -> trialNames.contains(t.getTrialName())) - .collect(Collectors.toList())); - } - - return trials; - } - - private List getTrialsByExRef(String referenceSource, String referenceId, Program program) throws ApiException { - Map cache = programExperimentCache.get(program.getId()); - List trials = new ArrayList<>(); - if (cache != null) { - trials.addAll(cache - .values() - .stream() - .filter(t -> { - BrAPIExternalReference xref = t - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS) - .equalsIgnoreCase(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI trial external reference found")); - return referenceId.equals(xref.getReferenceID()); - }) - .collect(Collectors.toList())); - } - - return trials; - } - - public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) - throws ApiException { - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - List createdTrials = new ArrayList<>(); - try { - if (!brAPITrialList.isEmpty()) { - Callable> postCallback = () -> { - List postedTrials = brAPIDAOUtil - .post(brAPITrialList, upload, api::trialsPost, importDAO::update); - return experimentById(postedTrials); - }; - createdTrials.addAll(programExperimentCache.post(programId, postCallback)); - } - - return createdTrials; - } catch (Exception e) { - throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); - } - } - public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException { - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - BrAPITrial updatedTrial = null; - try { - if (trial != null && !trialDbId.isBlank()) { - Callable> putCallback = () -> { - BrAPITrial putTrial = brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); - return experimentById(List.of(putTrial)); - }; - List cachedUpdates = programExperimentCache.post(programId, putCallback); - if (cachedUpdates.isEmpty()) { - throw new Exception(); - } - updatedTrial = cachedUpdates.get(0); - } - - return updatedTrial; - } catch (Exception e) { - throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); - } - } - - public List getTrials(UUID programId) throws ApiException { - return new ArrayList<>(programExperimentCache.get(programId).values()); - } - - //Removes program key from trial name - private List processExperimentsForDisplay( - List trials, - String programKey) throws ApiException { - List displayExperiments = new ArrayList<>(); - for (BrAPITrial trial: trials) { - trial.setTrialName(Utilities.removeProgramKey(trial.getTrialName(), programKey, "")); - displayExperiments.add(trial); - } - return displayExperiments; - } - - public Optional getTrialById(UUID programId, UUID trialId) throws ApiException, DoesNotExistException { - Map cache = programExperimentCache.get(programId); - BrAPITrial trial = null; - if (cache != null) { - trial = cache.get(trialId.toString()); - } - - return Optional.ofNullable(trial); - } - - public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { - List trials = getTrialsByDbIds(List.of(trialDbId), program); - return Utilities.getSingleOptional(trials); - } - - public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { - Map cache = programExperimentCache.get(program.getId()); - List trials = new ArrayList<>(); - if (cache != null) { - trials.addAll(cache - .values() - .stream() - .filter(t -> trialDbIds.contains(t.getTrialDbId())) - .collect(Collectors.toList())); - } - - return trials; - } - public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { - if(experimentIds.isEmpty()) { - return Collections.emptyList(); - } - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); - trialSearch.externalReferenceIDs(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - return brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); - } + List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException; } - - diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index 3ed181b75..5cbcc53a3 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -43,6 +43,7 @@ import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.DatabaseTest; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.impl.BrAPITrialDAOImpl; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.daos.impl.ImportMappingDAOImpl; @@ -54,6 +55,7 @@ import org.breedinginsight.dao.db.tables.pojos.ImporterImportEntity; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.daos.impl.ProgramDAOImpl; import org.breedinginsight.daos.impl.UserDAOImpl; import org.breedinginsight.model.BrAPIConstants; @@ -150,6 +152,8 @@ public class GigwaGenotypeServiceImplIntegrationTest extends DatabaseTest { @Inject private BrAPIEndpointProvider brAPIEndpointProvider; + @Inject + private ProgramCacheProvider cacheProvider; @Property(name = "gigwa.host") private String gigwaHost; @@ -186,11 +190,23 @@ ImportDAO importDAO() { return mock(ImportDAO.class); } + + @MockBean(BrAPITrialDAOImpl.class) + BrAPITrialDAO trialDAO() { + return mock(BrAPITrialDAOImpl.class); + } + +/* @MockBean(BrAPITrialDAO.class) BrAPITrialDAO trialDAO() { - return mock(BrAPITrialDAO.class); + return spy(new BrAPITrialDAO(cacheProvider, programDAO, importDAO, brAPIDAOUtil, mock(ProgramService.class),referenceSource , brAPIEndpointProvider, false)); } + */ + + + + @MockBean(BrAPIDAOUtil.class) BrAPIDAOUtil brAPIDAOUtil() { return spy(new BrAPIDAOUtil(1000, Duration.of(10, ChronoUnit.MINUTES), 1000, 100)); @@ -230,7 +246,7 @@ public GigwaGenotypeServiceImplIntegrationTest() { gigwa = new GenericContainer<>("breedinginsight/gigwa:develop") .withNetwork(super.getNetwork()) .withNetworkAliases("gigwa") - .withImagePullPolicy(PullPolicy.defaultPolicy()) + .withImagePullPolicy(PullPolicy.alwaysPull()) .withExposedPorts(8080) .withEnv("MONGO_IP", "gigwa_db") .withEnv("MONGO_PORT", "27017") @@ -285,6 +301,9 @@ public void setup() throws IllegalAccessException, NoSuchFieldException { storageService = applicationContext.getBean(SimpleStorageService.class, Qualifiers.byName("genotype")); storageService.createBucket(); + //cacheProvider = new ProgramCacheProvider(super.getRedisConnection()); + //trialDAO = new BrAPITrialDAO(cacheProvider, programDAO, importDAO, brAPIDAOUtil, mock(ProgramService.class),System.getenv("BRAPI_REFERENCE_SOURCE") , new BrAPIEndpointProvider()); + //trialDAO = Mockito.spy(new BrAPITrialDAO(cacheProvider, programDAO, importDAO, brAPIDAOUtil, mock(ProgramService.class),referenceSource , new BrAPIEndpointProvider())); } @AfterAll @@ -353,6 +372,8 @@ public void testFetchGermplasmGenotype() throws AuthorizationException, ApiExcep BrAPITrial trial = new BrAPITrial().externalReferences(List.of(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) .referenceID(UUID.randomUUID() .toString()))); + + doReturn(List.of(trial)).when(trialDAO).getTrials(any(UUID.class)); doAnswer(invocation -> { From 7025e3d0fb6f3f317accc6642f17470ca44214e4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:21:10 -0400 Subject: [PATCH 14/16] use trial name sans program key when fetching existing brapi data for import --- .../importer/services/processors/ExperimentProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f9c79bdd9..834394af1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1141,7 +1141,7 @@ private Map> initializeTrialByNameNoScop List uniqueTrialNames = experimentImportRows.stream() .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(experimentImport -> Utilities.appendProgramKey(experimentImport.getExpTitle(), programKey)) + .map(ExperimentObservation::getExpTitle) .distinct() .collect(Collectors.toList()); try { From 81313ac452a9724f64b78fdd9f28cd22af0d2013 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:32:39 -0400 Subject: [PATCH 15/16] create BrAPITrialDAOImpl --- .../importer/daos/impl/BrAPITrialDAOImpl.java | 282 ++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java b/src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java new file mode 100644 index 000000000..f26d039ba --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java @@ -0,0 +1,282 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.daos.impl; + +import io.micronaut.context.annotation.Context; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.core.TrialsApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +@Slf4j +@Context +@Singleton +public class BrAPITrialDAOImpl implements BrAPITrialDAO { + private final ProgramCache programExperimentCache; + private final ProgramDAO programDAO; + private final ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final ProgramService programService; + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final String referenceSource; + private final boolean runScheduledTasks; + + @Inject + public BrAPITrialDAOImpl(ProgramCacheProvider programCacheProvider, + ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + ProgramService programService, + @Property(name = "brapi.server.reference-source") String referenceSource, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks) { + this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiments, BrAPITrial.class); + this.programDAO = programDAO; + this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.programService = programService; + this.referenceSource = referenceSource; + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.runScheduledTasks = runScheduledTasks; + } + + + @Scheduled(initialDelay = "2s") + public void setup() { + if(!runScheduledTasks) { + return; + } + + // Populate the experiment cache for all programs on startup + log.debug("populating experiment cache"); + List programs = programDAO.getActive(); + if (programs != null) { + programExperimentCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + private Map fetchProgramExperiments(UUID programId) throws ApiException { + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + + // Get the program + List programs = programDAO.get(programId); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given key"); + } + Program program = programs.get(0); + + // Get the program experiments + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.externalReferenceIDs(List.of(programId.toString())); + trialSearch.externalReferenceSources( + List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + ); + List programExperiments = brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + + return experimentById(processExperimentsForDisplay(programExperiments, program.getKey())); + } + + private Map experimentById(List trials) { + Map experimentById = new HashMap<>(); + for (BrAPITrial experiment: trials) { + BrAPIExternalReference xref = experiment + .getExternalReferences() + .stream() + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS).equalsIgnoreCase(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + experimentById.put(xref.getReferenceID(), experiment); + } + return experimentById; + } + + @Override + public List getTrialsByName(List trialNames, Program program) throws ApiException { + Map cache = programExperimentCache.get(program.getId()); + List trials = new ArrayList<>(); + if (cache != null) { + + // TODO: replace with more performant cache search, e.g. RediSearch + trials.addAll(cache + .values() + .stream() + .filter(t -> trialNames.contains(t.getTrialName())) + .collect(Collectors.toList())); + } + + return trials; + } + + private List getTrialsByExRef(String referenceSource, String referenceId, Program program) throws ApiException { + Map cache = programExperimentCache.get(program.getId()); + List trials = new ArrayList<>(); + if (cache != null) { + trials.addAll(cache + .values() + .stream() + .filter(t -> { + BrAPIExternalReference xref = t + .getExternalReferences() + .stream() + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS) + .equalsIgnoreCase(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI trial external reference found")); + return referenceId.equals(xref.getReferenceID()); + }) + .collect(Collectors.toList())); + } + + return trials; + } + + @Override + public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) + throws ApiException { + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + List createdTrials = new ArrayList<>(); + try { + if (!brAPITrialList.isEmpty()) { + Callable> postCallback = () -> { + List postedTrials = brAPIDAOUtil + .post(brAPITrialList, upload, api::trialsPost, importDAO::update); + return experimentById(postedTrials); + }; + createdTrials.addAll(programExperimentCache.post(programId, postCallback)); + } + + return createdTrials; + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + @Override + public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException { + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + BrAPITrial updatedTrial = null; + try { + if (trial != null && !trialDbId.isBlank()) { + Callable> putCallback = () -> { + BrAPITrial putTrial = brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); + return experimentById(List.of(putTrial)); + }; + List cachedUpdates = programExperimentCache.post(programId, putCallback); + if (cachedUpdates.isEmpty()) { + throw new Exception(); + } + updatedTrial = cachedUpdates.get(0); + } + + return updatedTrial; + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + + @Override + public List getTrials(UUID programId) throws ApiException { + return new ArrayList<>(programExperimentCache.get(programId).values()); + } + + //Removes program key from trial name + private List processExperimentsForDisplay( + List trials, + String programKey) throws ApiException { + List displayExperiments = new ArrayList<>(); + for (BrAPITrial trial: trials) { + trial.setTrialName(Utilities.removeProgramKey(trial.getTrialName(), programKey, "")); + displayExperiments.add(trial); + } + return displayExperiments; + } + + @Override + public Optional getTrialById(UUID programId, UUID trialId) throws ApiException, DoesNotExistException { + Map cache = programExperimentCache.get(programId); + BrAPITrial trial = null; + if (cache != null) { + trial = cache.get(trialId.toString()); + } + + return Optional.ofNullable(trial); + } + + @Override + public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { + List trials = getTrialsByDbIds(List.of(trialDbId), program); + return Utilities.getSingleOptional(trials); + } + + @Override + public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { + Map cache = programExperimentCache.get(program.getId()); + List trials = new ArrayList<>(); + if (cache != null) { + trials.addAll(cache + .values() + .stream() + .filter(t -> trialDbIds.contains(t.getTrialDbId())) + .collect(Collectors.toList())); + } + + return trials; + } + @Override + public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { + if(experimentIds.isEmpty()) { + return Collections.emptyList(); + } + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); + trialSearch.externalReferenceIDs(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); + return brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + } +} + + From 3945b2c5d68397f1518486f34252b0bf0b2d2ab0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:17:55 -0400 Subject: [PATCH 16/16] clean up test --- ...gwaGenotypeServiceImplIntegrationTest.java | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index 5cbcc53a3..69f21d272 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -43,9 +43,9 @@ import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.DatabaseTest; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; -import org.breedinginsight.brapps.importer.daos.impl.BrAPITrialDAOImpl; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.daos.impl.BrAPITrialDAOImpl; import org.breedinginsight.brapps.importer.daos.impl.ImportMappingDAOImpl; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -55,7 +55,6 @@ import org.breedinginsight.dao.db.tables.pojos.ImporterImportEntity; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.UserDAO; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.daos.impl.ProgramDAOImpl; import org.breedinginsight.daos.impl.UserDAOImpl; import org.breedinginsight.model.BrAPIConstants; @@ -152,8 +151,6 @@ public class GigwaGenotypeServiceImplIntegrationTest extends DatabaseTest { @Inject private BrAPIEndpointProvider brAPIEndpointProvider; - @Inject - private ProgramCacheProvider cacheProvider; @Property(name = "gigwa.host") private String gigwaHost; @@ -196,16 +193,6 @@ BrAPITrialDAO trialDAO() { return mock(BrAPITrialDAOImpl.class); } -/* - @MockBean(BrAPITrialDAO.class) - BrAPITrialDAO trialDAO() { - return spy(new BrAPITrialDAO(cacheProvider, programDAO, importDAO, brAPIDAOUtil, mock(ProgramService.class),referenceSource , brAPIEndpointProvider, false)); - } - - */ - - - @MockBean(BrAPIDAOUtil.class) BrAPIDAOUtil brAPIDAOUtil() { @@ -301,10 +288,7 @@ public void setup() throws IllegalAccessException, NoSuchFieldException { storageService = applicationContext.getBean(SimpleStorageService.class, Qualifiers.byName("genotype")); storageService.createBucket(); - //cacheProvider = new ProgramCacheProvider(super.getRedisConnection()); - //trialDAO = new BrAPITrialDAO(cacheProvider, programDAO, importDAO, brAPIDAOUtil, mock(ProgramService.class),System.getenv("BRAPI_REFERENCE_SOURCE") , new BrAPIEndpointProvider()); - //trialDAO = Mockito.spy(new BrAPITrialDAO(cacheProvider, programDAO, importDAO, brAPIDAOUtil, mock(ProgramService.class),referenceSource , new BrAPIEndpointProvider())); - } + } @AfterAll public void teardown() {