diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 252d864aa..c8f42b303 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -51,6 +51,7 @@ public class BrAPIGermplasmDAO { private final ProgramDAO programDAO; private final ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Property(name = "brapi.server.reference-source") private String referenceSource; @@ -58,9 +59,10 @@ public class BrAPIGermplasmDAO { ProgramCache programGermplasmCache; @Inject - public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } @PostConstruct @@ -119,7 +121,7 @@ private Map fetchProgramGermplasm(UUID programId) throws BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); germplasmSearch.externalReferenceIDs(List.of(programId.toString())); germplasmSearch.externalReferenceSources(List.of(String.format("%s/programs", referenceSource))); - return processGermplasmForDisplay(BrAPIDAOUtil.search( + return processGermplasmForDisplay(brAPIDAOUtil.search( api::searchGermplasmPost, api::searchGermplasmSearchResultsDbIdGet, germplasmSearch @@ -211,7 +213,7 @@ private Map processGermplasmForDisplay(List importBrAPIGermplasm(List brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId)); try { - Callable> postFunction = () -> BrAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update); + Callable> postFunction = () -> brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update); return programGermplasmCache.post(programId, postFunction); } catch (ApiException e) { throw e; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java index e68175d9d..5a895d8ec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java @@ -32,14 +32,16 @@ public class BrAPICrossDAO { private BrAPIProvider brAPIProvider; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPICrossDAO(BrAPIProvider brAPIProvider) { + public BrAPICrossDAO(BrAPIProvider brAPIProvider, BrAPIDAOUtil brAPIDAOUtil) { this.brAPIProvider = brAPIProvider; + this.brAPIDAOUtil = brAPIDAOUtil; } public List createBrAPICrosses(List brAPICrossList) throws ApiException { CrossesApi api = brAPIProvider.getCrossesApi(BrAPIClientType.CORE); - return BrAPIDAOUtil.post(brAPICrossList, api::crossesPost); + return brAPIDAOUtil.post(brAPICrossList, api::crossesPost); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java index 635c5f4c4..11644a472 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java @@ -30,18 +30,20 @@ public class BrAPIListDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPIListDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPIListDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getListByName(List listNames, UUID programId) throws ApiException { BrAPIListSearchRequest listSearch = new BrAPIListSearchRequest(); listSearch.listNames(listNames); ListsApi api = new ListsApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchListsPost, api::searchListsSearchResultsDbIdGet, listSearch @@ -61,7 +63,7 @@ public List getListByTypeAndExternalRef(BrAPIListTypes listTyp .listType(listType); ListsApi api = new ListsApi(programDAO.getCoreClient(programId)); - return processListsForProgram(BrAPIDAOUtil.search( + return processListsForProgram(brAPIDAOUtil.search( api::searchListsPost, api::searchListsSearchResultsDbIdGet, searchRequest diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java index 30c73d3e5..d7819c366 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java @@ -36,11 +36,13 @@ public class BrAPILocationDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPILocationDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPILocationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getLocationsByName(List locationNames, UUID programId) throws ApiException { @@ -49,7 +51,7 @@ public List getLocationsByName(List locationNames, UUID p locationSearchRequest.setLocationNames(new ArrayList<>(locationNames)); //TODO: Locations don't connect to programs. How to get locations for the program? LocationsApi api = new LocationsApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchLocationsPost, api::searchLocationsSearchResultsDbIdGet, locationSearchRequest @@ -58,7 +60,7 @@ public List getLocationsByName(List locationNames, UUID p public List createBrAPILocation(List brAPILocationList, UUID programId, ImportUpload upload) throws ApiException { LocationsApi api = new LocationsApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.post(brAPILocationList, upload, api::locationsPost, importDAO::update); + return brAPIDAOUtil.post(brAPILocationList, upload, api::locationsPost, importDAO::update); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 2bcd6795c..e67415d32 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -45,12 +45,14 @@ public class BrAPIObservationDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; private UUID programId; @Inject - public BrAPIObservationDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPIObservationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getObservationsByStudyName(List studyNames, Program program) throws ApiException { @@ -60,7 +62,7 @@ public List getObservationsByStudyName(List studyNames observationSearchRequest.setStudyNames(new ArrayList<>(studyNames)); ObservationsApi api = new ObservationsApi(programDAO.getCoreClient(program.getId())); this.programId = program.getId(); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchObservationsPost, this::searchObservationsSearchResultsDbIdGet, observationSearchRequest @@ -76,7 +78,7 @@ public List getObservationsByStudyName(List studyNames public List createBrAPIObservation(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { ObservationsApi api = new ObservationsApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); + return brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index 4fba6e0b4..faa124273 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -38,11 +38,13 @@ public class BrAPIObservationUnitDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPIObservationUnitDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } /* @@ -69,7 +71,7 @@ public List getObservationUnitByName(List observat observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); observationUnitSearchRequest.observationUnitNames(observationUnitNames); ObservationUnitsApi api = new ObservationUnitsApi(programDAO.getCoreClient(program.getId())); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchObservationunitsPost, api::searchObservationunitsSearchResultsDbIdGet, observationUnitSearchRequest @@ -78,6 +80,6 @@ public List getObservationUnitByName(List observat public List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId, ImportUpload upload) throws ApiException { ObservationUnitsApi api = new ObservationUnitsApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); + return brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java index 2c51f4262..abeac1011 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java @@ -32,17 +32,19 @@ public class BrAPIObservationVariableDAO { private ProgramDAO programDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPIObservationVariableDAO(ProgramDAO programDAO) { + public BrAPIObservationVariableDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getVariableByName(List variableNames, UUID programId) throws ApiException { BrAPIObservationVariableSearchRequest variableSearch = new BrAPIObservationVariableSearchRequest(); variableSearch.observationVariableNames(variableNames); ObservationVariablesApi api = new ObservationVariablesApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchVariablesPost, api::searchVariablesSearchResultsDbIdGet, variableSearch diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index 3752a394b..474edbb38 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -38,11 +38,13 @@ public class BrAPIStudyDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getStudyByName(List studyNames, Program program) throws ApiException { @@ -50,7 +52,7 @@ public List getStudyByName(List studyNames, Program program) studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.studyNames(studyNames); StudiesApi api = new StudiesApi(programDAO.getCoreClient(program.getId())); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchStudiesPost, api::searchStudiesSearchResultsDbIdGet, studySearch @@ -63,7 +65,7 @@ public List getStudiesByExperimentID(UUID experimentID, Program prog studySearch.addExternalReferenceIDsItem(experimentID.toString()); studySearch.addExternalReferenceSourcesItem(BRAPI_REFERENCE_SOURCE + "/trials"); StudiesApi api = new StudiesApi(programDAO.getCoreClient(program.getId())); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchStudiesPost, api::searchStudiesSearchResultsDbIdGet, studySearch @@ -72,7 +74,7 @@ public List getStudiesByExperimentID(UUID experimentID, Program prog public List createBrAPIStudy(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = new StudiesApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); + return brAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 30b1a8d60..2e61401ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -35,11 +35,13 @@ public class BrAPITrialDAO { private ProgramDAO programDAO; private ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO) { + public BrAPITrialDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { this.programDAO = programDAO; this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getTrialByName(List trialNames, Program program) throws ApiException { @@ -47,7 +49,7 @@ public List getTrialByName(List trialNames, Program program) trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); trialSearch.trialNames(trialNames); TrialsApi api = new TrialsApi(programDAO.getCoreClient(program.getId())); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchTrialsPost, api::searchTrialsSearchResultsDbIdGet, trialSearch @@ -56,7 +58,7 @@ public List getTrialByName(List trialNames, Program program) public List createBrAPITrial(List brAPITrialList, UUID programId, ImportUpload upload) throws ApiException { TrialsApi api = new TrialsApi(programDAO.getCoreClient(programId)); - return BrAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); + return brAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); } } diff --git a/src/main/java/org/breedinginsight/daos/ObservationDAO.java b/src/main/java/org/breedinginsight/daos/ObservationDAO.java index 2dbaab866..86f8ccc1e 100644 --- a/src/main/java/org/breedinginsight/daos/ObservationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ObservationDAO.java @@ -39,10 +39,12 @@ public class ObservationDAO { private BrAPIProvider brAPIProvider; + private final BrAPIDAOUtil brAPIDAOUtil; @Inject - public ObservationDAO(BrAPIProvider brAPIProvider) { + public ObservationDAO(BrAPIProvider brAPIProvider, BrAPIDAOUtil brAPIDAOUtil) { this.brAPIProvider = brAPIProvider; + this.brAPIDAOUtil = brAPIDAOUtil; } public List getObservationsByVariableDbId(String observationVariableDbId) { @@ -67,7 +69,7 @@ public List getObservationsByVariableDbIds(List observ .observationVariableDbIds(observationVariableDbIds); ObservationsApi api = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchObservationsPost, this::searchObservationsSearchResultsDbIdGet, request @@ -86,7 +88,7 @@ public List getObservationsByVariableAndBrAPIProgram(String br .programDbIds(List.of(brapiProgramId)); ObservationsApi api = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchObservationsPost, this::searchObservationsSearchResultsDbIdGet, request diff --git a/src/main/java/org/breedinginsight/daos/TraitDAO.java b/src/main/java/org/breedinginsight/daos/TraitDAO.java index 27c755354..f929bdffe 100644 --- a/src/main/java/org/breedinginsight/daos/TraitDAO.java +++ b/src/main/java/org/breedinginsight/daos/TraitDAO.java @@ -56,17 +56,19 @@ public class TraitDAO extends TraitDao { @Property(name = "brapi.server.reference-source") private String referenceSource; private ObservationDAO observationDao; + private final BrAPIDAOUtil brAPIDAOUtil; private Gson gson; private final static String TAGS_KEY = "tags"; private final static String FULLNAME_KEY = "fullname"; @Inject - public TraitDAO(Configuration config, DSLContext dsl, BrAPIProvider brAPIProvider, ObservationDAO observationDao) { + public TraitDAO(Configuration config, DSLContext dsl, BrAPIProvider brAPIProvider, ObservationDAO observationDao, BrAPIDAOUtil brAPIDAOUtil) { super(config); this.dsl = dsl; this.brAPIProvider = brAPIProvider; this.observationDao = observationDao; + this.brAPIDAOUtil = brAPIDAOUtil; this.gson = new Gson(); } @@ -245,7 +247,7 @@ public List searchVariables(List variableIds) .externalReferenceIDs(variableIds); ObservationVariablesApi api = brAPIProvider.getVariablesAPI(PHENO); - return BrAPIDAOUtil.search( + return brAPIDAOUtil.search( api::searchVariablesPost, api::searchVariablesSearchResultsDbIdGet, request diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 360ebcd75..dc737c131 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -17,6 +17,7 @@ package org.breedinginsight.utilities; +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import io.reactivex.functions.Consumer; import io.reactivex.functions.Function; @@ -27,54 +28,85 @@ import org.brapi.v2.model.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import javax.inject.Singleton; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.concurrent.TimeUnit; +@Singleton public class BrAPIDAOUtil { - public static Integer SEARCH_WAIT_TIME = 1000; - public static Integer SEARCH_TIMEOUT = Long.valueOf(TimeUnit.MINUTES.toMillis(10)).intValue(); - public static Integer RESULTS_PER_QUERY = 100000; - public static Integer POST_GROUP_SIZE = 100; + @Property(name = "brapi.search.wait-time") + private int searchWaitTime; + @Property(name = "brapi.read-timeout") + private Duration searchTimeout; + @Property(name = "brapi.page-size") + private int pageSize; + @Property(name = "brapi.post-group-size") + private int postGroupSize; - public static List search(Function, Optional>>> searchMethod, + public List search(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, U searchBody ) throws ApiException { try { List listResult = new ArrayList<>(); - searchBody.pageSize(RESULTS_PER_QUERY); - ApiResponse, Optional>> response = - searchMethod.apply(searchBody); + searchBody.pageSize(pageSize); + ApiResponse, Optional>> response = searchMethod.apply(searchBody); if (response.getBody().getLeft().isPresent()) { BrAPIResponse listResponse = (BrAPIResponse) response.getBody().getLeft().get(); - BrAPIResponseResult responseResult = (BrAPIResponseResult) listResponse.getResult(); - listResult = responseResult != null ? responseResult.getData() : - new ArrayList<>(); - // TODO: Check that all of the pages were returned + listResult = getListResult(response); + + if(hasMorePages(listResponse)) { + int currentPage = listResponse.getMetadata().getPagination().getCurrentPage() + 1; + int totalPages = listResponse.getMetadata().getPagination().getTotalPages(); + + while (currentPage < totalPages) { + searchBody.setPage(currentPage); + response = searchMethod.apply(searchBody); + if (response.getBody().getLeft().isPresent()) { + listResult.addAll(getListResult(response)); + } + + currentPage++; + } + } } else { // Hit the get endpoint until we get a response Integer accruedWait = 0; Boolean searchFinished = false; + int currentPage = 0; while (!searchFinished) { BrAPIAcceptedSearchResponse searchResult = response.getBody().getRight().get(); - // TODO: Check if we have more to get for pages - ApiResponse, Optional>> searchGetResponse = searchGetMethod.apply(searchResult.getResult().getSearchResultsDbId(), 0, RESULTS_PER_QUERY); + ApiResponse, Optional>> searchGetResponse = searchGetMethod.apply(searchResult.getResult().getSearchResultsDbId(), currentPage, pageSize); if (searchGetResponse.getBody().getLeft().isPresent()) { searchFinished = true; BrAPIResponse listResponse = (BrAPIResponse) searchGetResponse.getBody().getLeft().get(); - BrAPIResponseResult responseResult = (BrAPIResponseResult) listResponse.getResult(); - listResult = responseResult != null ? responseResult.getData() : - new ArrayList<>(); + listResult = getListResult(searchGetResponse); + + if(hasMorePages(listResponse)) { + currentPage++; + int totalPages = listResponse.getMetadata() + .getPagination() + .getTotalPages(); + + while (currentPage < totalPages) { + searchGetResponse = searchGetMethod.apply(searchResult.getResult().getSearchResultsDbId(), currentPage, pageSize); + if (searchGetResponse.getBody().getLeft().isPresent()) { + listResult.addAll(getListResult(searchGetResponse)); + } + + currentPage++; + } + } } else { // Wait a bit before we call again - Thread.sleep(SEARCH_WAIT_TIME); - accruedWait += SEARCH_WAIT_TIME; - if (accruedWait >= SEARCH_TIMEOUT) { + Thread.sleep(searchWaitTime); + accruedWait += searchWaitTime; + if (accruedWait >= searchTimeout.toMillis()) { throw new ApiException("Search response timeout"); } } @@ -89,7 +121,20 @@ public static List searc } } - public static List post(List brapiObjects, + private boolean hasMorePages(BrAPIResponse listResponse) { + return listResponse.getMetadata() != null + && listResponse.getMetadata().getPagination() != null + && listResponse.getMetadata().getPagination().getCurrentPage() < listResponse.getMetadata().getPagination().getTotalPages() - 1; + } + + private List getListResult(ApiResponse, Optional>> searchGetResponse) { + BrAPIResponse listResponse = (BrAPIResponse) searchGetResponse.getBody().getLeft().get(); + BrAPIResponseResult responseResult = (BrAPIResponseResult) listResponse.getResult(); + return responseResult != null ? responseResult.getData() : + new ArrayList<>(); + } + + public List post(List brapiObjects, ImportUpload upload, Function, ApiResponse> postMethod, Consumer progressUpdateMethod) throws ApiException { @@ -102,8 +147,8 @@ public static List post(List brapiObjects, Integer finished = upload != null && upload.getProgress().getFinished() != null ? Math.toIntExact(upload.getProgress().getFinished()) : 0; while (currentRightBorder < brapiObjects.size()) { - List postChunk = brapiObjects.size() > (currentRightBorder + POST_GROUP_SIZE) ? - brapiObjects.subList(currentRightBorder, currentRightBorder + POST_GROUP_SIZE) : + List postChunk = brapiObjects.size() > (currentRightBorder + postGroupSize) ? + brapiObjects.subList(currentRightBorder, currentRightBorder + postGroupSize) : brapiObjects.subList(currentRightBorder, brapiObjects.size()); // Update our progress in the db if (upload != null) { @@ -121,7 +166,7 @@ public static List post(List brapiObjects, if (data.size() != postChunk.size()) throw new ApiException("Number of brapi objects returned does not equal number sent"); listResult.addAll(data); finished += data.size(); - currentRightBorder += POST_GROUP_SIZE; + currentRightBorder += postGroupSize; } if (upload != null) { @@ -137,7 +182,7 @@ public static List post(List brapiObjects, } } - public static List post(List brapiObjects, + public List post(List brapiObjects, Function, ApiResponse> postMethod) throws ApiException { return post(brapiObjects, null, postMethod, null); } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c1d207a0e..2c4f8179e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -159,6 +159,10 @@ brapi: geno-url: ${brapi.server.default-url} reference-source: ${BRAPI_REFERENCE_SOURCE:breedinginsight.org} read-timeout: 10m + page-size: 1000 + search: + wait-time: 1000 + post-group-size: 100 email: relay-server: diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index 9e0493760..758c613c3 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -1,6 +1,8 @@ package org.breedinginsight.services; import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Property; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.functions.Function; import io.reactivex.functions.Function3; import lombok.SneakyThrows; @@ -13,6 +15,7 @@ import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; @@ -26,6 +29,7 @@ import tech.tablesaw.api.Table; import java.io.InputStream; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -42,18 +46,25 @@ public class BrAPIGermplasmServiceUnitTest { private ProgramService programService; private ProgramDAO programDAO; private BrAPIGermplasmService germplasmService; + private BrAPIDAOUtil brAPIDAOUtil; + private String referenceSource; + @SneakyThrows @BeforeEach void setup() { //Set up mocks in here + referenceSource = "breedinginsight.org"; listDAO = mock(BrAPIListDAO.class); programDAO = mock(ProgramDAO.class); - germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class)); + brAPIDAOUtil = mock(BrAPIDAOUtil.class); + germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class), brAPIDAOUtil); programService = mock(ProgramService.class); + + Field externalReferenceSource = BrAPIGermplasmDAO.class.getDeclaredField("referenceSource"); + externalReferenceSource.setAccessible(true); + externalReferenceSource.set(germplasmDAO, referenceSource); } - //TODO: Refactoring in BI-1443 needs to be done before this can be enabled due to BrAPIDAOUtil.search currently being static - /* @Test @SneakyThrows public void getGermplasmListExport() { @@ -68,7 +79,7 @@ public void getGermplasmListExport() { BrAPIListsSingleResponse listResponse = new BrAPIListsSingleResponse(); String listId = "1"; String listName = "List Name"; - List germplasmNames = Arrays.asList("Germplasm A", "Germplasm B"); + List germplasmNames = Arrays.asList("Germplasm A [TEST-1]", "Germplasm B [TEST-2]"); BrAPIListDetails listDetails = new BrAPIListDetails(); listDetails.setData(germplasmNames); listDetails.setListName(listName + " [TEST-germplasm]"); @@ -91,7 +102,8 @@ public void getGermplasmListExport() { testGermplasm.setAdditionalInfo(additionalInfo); List externalRef = new ArrayList<>(); BrAPIExternalReference testReference = new BrAPIExternalReference(); - testReference.setReferenceSource("TestSource"); + testReference.setReferenceSource(referenceSource); + testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); germplasm.add(testGermplasm); @@ -106,30 +118,38 @@ public void getGermplasmListExport() { additionalInfo.addProperty("importEntryNumber", "3"); additionalInfo.addProperty("breedingMethod", "Autopolyploid"); testGermplasm.setAdditionalInfo(additionalInfo); + testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(referenceSource); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef = new ArrayList<>(); + externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); germplasm.add(testGermplasm); - //Create stubs + //Stub out the spies BrAPIListDAO brAPIListSpy = spy(listDAO); - ProgramService programSpy = spy(programService); - - when(programDAO.getAll()).thenReturn(Arrays.asList(Program.builder().id(UUID.randomUUID()).name("Test Program").build())); - doReturn(listResponse).when(brAPIListSpy).getListById(listId, testProgramId); - MockedStatic brapiDaoUtilMock = mockStatic(BrAPIDAOUtil.class); - brapiDaoUtilMock.when(() -> BrAPIDAOUtil.search(any(Function.class), - any(Function3.class), - any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); + ProgramService programSpy = spy(programService); doReturn(Optional.of(testProgram)).when(programSpy).getById(testProgramId); + //Stub out the mocks + when(programDAO.getAll()).thenReturn(Arrays.asList(Program.builder().id(testProgramId).name("Test Program").active(true).build())); + when(programDAO.fetchOneById(any(UUID.class))).thenReturn(testProgram); + when(programDAO.get(any(UUID.class))).thenReturn(Arrays.asList(testProgram)); + when(brAPIDAOUtil.search(any(Function.class), + any(Function3.class), + any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); + + //Create germplasm cache of stub data Method setupMethod = BrAPIGermplasmDAO.class.getDeclaredMethod("setup"); setupMethod.setAccessible(true); setupMethod.invoke(germplasmDAO); + //Create test instance of service, injecting spy- and mock-dependencies germplasmService = new BrAPIGermplasmService(brAPIListSpy, programSpy, germplasmDAO); //Retrieve file - DownloadFile downloadFile = germplasmService.exportGermplasmList(testProgramId, listId); + DownloadFile downloadFile = germplasmService.exportGermplasmList(testProgramId, listId, FileType.XLS); InputStream inputStream = downloadFile.getStreamedFile().getInputStream(); Table resultTable = FileUtil.parseTableFromExcel(inputStream, 0); @@ -141,10 +161,9 @@ public void getGermplasmListExport() { //Check file values assertEquals(listName+"_"+timestamp, downloadFile.getFileName(), "Incorrect export file name"); - assertEquals(2, resultTable.rowCount(), "Wrong number of rows were exported"); assertEquals(expectedColumnNames, resultTable.columnNames(), "Incorrect columns were exported"); + assertEquals(2, resultTable.rowCount(), "Wrong number of rows were exported"); assertEquals("Germplasm A", resultTable.get(0, 1), "Incorrect data exported"); assertEquals("2", resultTable.get(0, 6), "Incorrect data exported"); } - */ }