diff --git a/docker-compose.yml b/docker-compose.yml index 6414d305a..a19472585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,8 @@ services: depends_on: - bidb - brapi-server + - redis + - mailhog volumes: - /usr/bin/docker:/usr/bin/docker - /var/run/docker.sock:/var/run/docker.sock @@ -51,6 +53,9 @@ services: - EMAIL_RELAY_HOST=${EMAIL_RELAY_HOST} - EMAIL_RELAY_PORT=${EMAIL_RELAY_PORT} - EMAIL_FROM=${EMAIL_FROM} + - REDIS_URL=${REDIS_URL:-redis://redis:6379} + - REDIS_TIMEOUT=${REDIS_TIMEOUT:-30s} + - REDIS_SSL=${REDIS_SSL:-false} ports: - ${API_INTERNAL_PORT}:${API_INTERNAL_PORT} networks: @@ -62,7 +67,7 @@ services: - POSTGRES_DB=${DB_NAME} - POSTGRES_PASSWORD=${DB_PASSWORD} ports: - - 5432:5432 + - "5432:5432" volumes: - biapi_data:/var/lib/postgresql/data networks: @@ -70,7 +75,7 @@ services: aliases: - dbserver brapi-server: - image: breedinginsight/brapi-java-server:latest + image: breedinginsight/brapi-java-server:develop container_name: brapi-server depends_on: - bidb @@ -93,8 +98,16 @@ services: container_name: mailhog restart: always ports: - - 1025:1025 - - 8025:8025 + - "1025:1025" + - "8025:8025" + redis: + image: redis + container_name: redis + restart: always + ports: + - "6379:6379" + networks: + backend: networks: backend: diff --git a/io-micronaut/jar_files.zip b/io-micronaut/jar_files.zip deleted file mode 100644 index 8030fe766..000000000 Binary files a/io-micronaut/jar_files.zip and /dev/null differ diff --git a/io-micronaut/jar_files/aop-1.0.0.RC2.jar b/io-micronaut/jar_files/aop-1.0.0.RC2.jar deleted file mode 100644 index 00861b268..000000000 Binary files a/io-micronaut/jar_files/aop-1.0.0.RC2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/core-1.0.0.RC2.jar b/io-micronaut/jar_files/core-1.0.0.RC2.jar deleted file mode 100644 index 5a43c117d..000000000 Binary files a/io-micronaut/jar_files/core-1.0.0.RC2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/http-1.0.0.RC2.jar b/io-micronaut/jar_files/http-1.0.0.RC2.jar deleted file mode 100644 index de7200e29..000000000 Binary files a/io-micronaut/jar_files/http-1.0.0.RC2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/inject-1.0.0.RC2.jar b/io-micronaut/jar_files/inject-1.0.0.RC2.jar deleted file mode 100644 index 7d31dbd0b..000000000 Binary files a/io-micronaut/jar_files/inject-1.0.0.RC2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/jackson-annotations-2.9.0.jar b/io-micronaut/jar_files/jackson-annotations-2.9.0.jar deleted file mode 100644 index c602d75d4..000000000 Binary files a/io-micronaut/jar_files/jackson-annotations-2.9.0.jar and /dev/null differ diff --git a/io-micronaut/jar_files/jackson-core-2.9.7.jar b/io-micronaut/jar_files/jackson-core-2.9.7.jar deleted file mode 100644 index fa46a7f6a..000000000 Binary files a/io-micronaut/jar_files/jackson-core-2.9.7.jar and /dev/null differ diff --git a/io-micronaut/jar_files/jackson-databind-2.9.7.jar b/io-micronaut/jar_files/jackson-databind-2.9.7.jar deleted file mode 100644 index 76d50b4f2..000000000 Binary files a/io-micronaut/jar_files/jackson-databind-2.9.7.jar and /dev/null differ diff --git a/io-micronaut/jar_files/jackson-datatype-jdk8-2.9.7.jar b/io-micronaut/jar_files/jackson-datatype-jdk8-2.9.7.jar deleted file mode 100644 index f1511e57f..000000000 Binary files a/io-micronaut/jar_files/jackson-datatype-jdk8-2.9.7.jar and /dev/null differ diff --git a/io-micronaut/jar_files/jackson-datatype-jsr310-2.9.7.jar b/io-micronaut/jar_files/jackson-datatype-jsr310-2.9.7.jar deleted file mode 100644 index 6993ab2aa..000000000 Binary files a/io-micronaut/jar_files/jackson-datatype-jsr310-2.9.7.jar and /dev/null differ diff --git a/io-micronaut/jar_files/javax.annotation-api-1.3.2.jar b/io-micronaut/jar_files/javax.annotation-api-1.3.2.jar deleted file mode 100644 index a8a470a71..000000000 Binary files a/io-micronaut/jar_files/javax.annotation-api-1.3.2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/javax.inject-1.jar b/io-micronaut/jar_files/javax.inject-1.jar deleted file mode 100644 index b2a9d0bf7..000000000 Binary files a/io-micronaut/jar_files/javax.inject-1.jar and /dev/null differ diff --git a/io-micronaut/jar_files/jsr305-3.0.2.jar b/io-micronaut/jar_files/jsr305-3.0.2.jar deleted file mode 100644 index 59222d9ca..000000000 Binary files a/io-micronaut/jar_files/jsr305-3.0.2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/reactive-streams-1.0.2.jar b/io-micronaut/jar_files/reactive-streams-1.0.2.jar deleted file mode 100644 index 8e8a9ce0c..000000000 Binary files a/io-micronaut/jar_files/reactive-streams-1.0.2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/runtime-1.0.0.RC2.jar b/io-micronaut/jar_files/runtime-1.0.0.RC2.jar deleted file mode 100644 index e85e414fa..000000000 Binary files a/io-micronaut/jar_files/runtime-1.0.0.RC2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/rxjava-2.2.2.jar b/io-micronaut/jar_files/rxjava-2.2.2.jar deleted file mode 100644 index 18959b2f3..000000000 Binary files a/io-micronaut/jar_files/rxjava-2.2.2.jar and /dev/null differ diff --git a/io-micronaut/jar_files/slf4j-api-1.7.25.jar b/io-micronaut/jar_files/slf4j-api-1.7.25.jar deleted file mode 100644 index 0143c0996..000000000 Binary files a/io-micronaut/jar_files/slf4j-api-1.7.25.jar and /dev/null differ diff --git a/io-micronaut/jar_files/snakeyaml-1.23.jar b/io-micronaut/jar_files/snakeyaml-1.23.jar deleted file mode 100644 index adcef4f72..000000000 Binary files a/io-micronaut/jar_files/snakeyaml-1.23.jar and /dev/null differ diff --git a/io-micronaut/jar_files/validation-api-2.0.1.Final.jar b/io-micronaut/jar_files/validation-api-2.0.1.Final.jar deleted file mode 100644 index 2368e10a5..000000000 Binary files a/io-micronaut/jar_files/validation-api-2.0.1.Final.jar and /dev/null differ diff --git a/pom.xml b/pom.xml index 79b24ff51..e31e82f74 100644 --- a/pom.xml +++ b/pom.xml @@ -59,9 +59,9 @@ UTF-8 UTF-8 org.breedinginsight.api.Application - 2.19.1 - 2.19.1 - --enable-preview + 2.22.2 + 2.22.2 + -Xmx1024m -XX:MaxPermSize=256m --enable-preview 3.16.3 42.3.2 @@ -395,6 +395,11 @@ cloning ${cloning.version} + + org.redisson + redisson-micronaut-20 + 3.17.5 + @@ -571,21 +576,21 @@ org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-plugin.version} - - - org.junit.platform - junit-platform-surefire-provider - 1.1.0 - - - org.junit.jupiter - junit-jupiter-engine - 5.7.0 - - + + + + + + + + + + + + false - ${jvm.options} + ${jvm.options} true %regex[.*] diff --git a/src/main/java/org/breedinginsight/brapi/v1/services/BrapiObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v1/services/BrapiObservationVariableService.java index 44e7ff248..6c65aaaa1 100644 --- a/src/main/java/org/breedinginsight/brapi/v1/services/BrapiObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v1/services/BrapiObservationVariableService.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapi.v1.services; import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapi.v1.model.*; import org.breedinginsight.dao.db.enums.DataType; @@ -111,8 +112,15 @@ private ObservationVariable mapBiTraitToBrapiV1ObservationVariable(org.breedingi TraitDataType dataType = mapBiDataTypeToBrapiV1TraitDataType(biScale.getDataType()); - List categories = biScale.getCategories().stream() - .map(category -> category.getValue()).collect(Collectors.toList()); + + List categories = new ArrayList<>(); + + if(biScale.getCategories() != null) { + categories = biScale.getCategories() + .stream() + .map(BrAPIScaleValidValuesCategories::getValue) + .collect(Collectors.toList()); + } ValidValues validValues = ValidValues.builder() .categories(categories) @@ -169,7 +177,7 @@ private ObservationVariable mapBiTraitToBrapiV1ObservationVariable(org.breedingi //.scientist() // missing from bi trait model but stored in BrAPI service //.status(trait.getStatus()) .submissionTimestamp(trait.getCreatedAt()) - .synonyms(trait.getSynonyms().isEmpty() ? synonyms : trait.getSynonyms()) // TODO: fix need to have synonym for field book bug + .synonyms(trait.getSynonyms() == null || trait.getSynonyms().isEmpty() ? synonyms : trait.getSynonyms()) // TODO: fix need to have synonym for field book bug .trait(brapiTrait) .xref(trait.getId().toString()) .observationVariableDbId(trait.getId().toString()) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index 9f3792b2f..a2a966289 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -27,6 +27,7 @@ import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index c8f42b303..dede1e25c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -21,23 +21,28 @@ import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.germplasm.GermplasmQueryParams; import org.brapi.client.v2.modules.germplasm.GermplasmApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; +import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; -import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; @@ -56,20 +61,30 @@ public class BrAPIGermplasmDAO { @Property(name = "brapi.server.reference-source") private String referenceSource; - ProgramCache programGermplasmCache; + @Property(name = "micronaut.bi.api.run-scheduled-tasks") + private boolean runScheduledTasks; + + private final ProgramCache programGermplasmCache; @Inject - public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) { + public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramCacheProvider programCacheProvider) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; + this.programGermplasmCache = programCacheProvider.getProgramCache(this::fetchProgramGermplasm, BrAPIGermplasm.class); } - @PostConstruct - private void setup() { + @Scheduled(initialDelay = "2s") + public void setup() { + if(!runScheduledTasks) { + return; + } // Populate germplasm cache for all programs on startup - List programs = programDAO.getAll().stream().filter(Program::getActive).map(Program::getId).collect(Collectors.toList()); - programGermplasmCache = new ProgramCache<>(this::fetchProgramGermplasm, programs); + log.debug("populating germplasm cache"); + List programs = programDAO.getActive(); + if(programs != null) { + programGermplasmCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } } /** @@ -111,10 +126,11 @@ public List getRawGermplasm(UUID programId) throws ApiException */ private Map fetchProgramGermplasm(UUID programId) throws ApiException { GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId)); - // Get the program key List programs = programDAO.get(programId); - if (programs.size() != 1) throw new InternalServerException("Program was not found for given key"); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given key"); + } Program program = programs.get(0); // Set query params and make call @@ -137,7 +153,7 @@ private Map fetchProgramGermplasm(UUID programId) throws private Map processGermplasmForDisplay(List programGermplasm, String programKey) { // Process the germplasm Map programGermplasmMap = new HashMap<>(); - log.debug("processing germ for display: " + programGermplasm); + log.trace("processing germ for display: " + programGermplasm); Map programGermplasmByFullName = new HashMap<>(); for (BrAPIGermplasm germplasm: programGermplasm) { programGermplasmByFullName.put(germplasm.getGermplasmName(), germplasm); @@ -212,13 +228,15 @@ private Map processGermplasmForDisplay(List importBrAPIGermplasm(List brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException { GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId)); + var program = programDAO.fetchOneById(programId); try { - Callable> postFunction = () -> brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update); + Callable> postFunction = () -> { + List postResponse = brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update); + return processGermplasmForDisplay(postResponse, program.getKey()); + }; return programGermplasmCache.post(programId, postFunction); - } catch (ApiException e) { - throw e; } catch (Exception e) { - throw new InternalServerException("Unknown error has occurred: " + e.getMessage()); + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/ProgramCache.java b/src/main/java/org/breedinginsight/brapi/v2/dao/ProgramCache.java deleted file mode 100644 index 0b31ff8ad..000000000 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/ProgramCache.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapi.v2.dao; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.rits.cloning.Cloner; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import javax.validation.constraints.NotNull; - -import java.util.*; -import java.util.concurrent.*; -import java.util.stream.Collectors; - -/** - * - * @param key/id of object - * @param object - */ -@Slf4j -public class ProgramCache { - - private final FetchFunction> fetchMethod; - private final Map programSemaphore = new HashMap<>(); - private final Cloner cloner; - - private final Executor executor = Executors.newCachedThreadPool(); - private final LoadingCache> cache = CacheBuilder.newBuilder() - .build(new CacheLoader<>() { - @Override - public Map load(@NotNull UUID programId) throws Exception { - try { - Map values = fetchMethod.apply(programId); - log.debug("cache loading complete.\nprogramId: " + programId); - return values; - } catch (Exception e) { - log.error("cache loading error:\nprogramId: " + programId, e); - cache.invalidate(programId); - throw e; - } - } - }); - - public ProgramCache(FetchFunction> fetchMethod, List keys) { - this.fetchMethod = fetchMethod; - this.cloner = new Cloner(); - // Populate cache on start up - for (UUID key: keys) { - updateCache(key); - } - } - - public ProgramCache(FetchFunction> fetchMethod) { - this.fetchMethod = fetchMethod; - this.cloner = new Cloner(); - } - - public Map get(UUID programId) throws ApiException { - try { - // This will get current cache data, or wait for the refresh to finish if there is no cache data. - // TODO: Do we want to wait for a refresh method if it is running? Returns current data right now, even if old - if (!programSemaphore.containsKey(programId) || cache.getIfPresent(programId) == null) { - // If the cache is missing, refresh and get - log.trace("cache miss, fetching from source.\nprogramId: " + programId); - updateCache(programId); - Map result = new HashMap<>(cache.get(programId)); - result = result.entrySet().stream().map(cloner::deepClone) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return result; - } else { - log.trace("cache contains records for the program.\nprogramId: " + programId); - // Most cases where the cache is populated - Map result = new HashMap<>(cache.get(programId)); - result = result.entrySet().stream().map(cloner::deepClone) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return result; - } - } catch (ExecutionException e) { - log.error("cache error:\nprogramId: " + programId, e); - return fetchMethod.apply(programId); - } - } - - /* - Checks to see whether atleast 1 refresh process is queued up and doesn't make another - refresh request if there is one queued. The idea here is that if you request to refresh the - cache state, but there is a refresh already waiting, that waiting refresh will grab the most recent - state of the cache, so queuing another one will be a waste of threads. - */ - private void updateCache(UUID programId) { - - if (!programSemaphore.containsKey(programId)) { - programSemaphore.put(programId, new Semaphore(1)); - } - - if (!programSemaphore.get(programId).hasQueuedThreads()) { // if false, an update is already queued, skip this one - // Start a refresh process asynchronously - executor.execute(() -> { - // Synchronous - try { - programSemaphore.get(programId).acquire(); - cache.refresh(programId); - } catch (InterruptedException e) { - log.error("cache loading error:\nprogramId: " + programId, e); - throw new InternalServerException(e.getMessage(), e); - } finally { - programSemaphore.get(programId).release(); - } - }); - } - } - - public List post(UUID programId, Callable> postMethod) throws Exception { - List response = postMethod.call(); - updateCache(programId); - return response; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java index 11644a472..7ea484284 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java @@ -12,13 +12,12 @@ import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.request.BrAPIListSearchRequest; -import org.brapi.v2.model.core.response.BrAPIListsListResponse; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.ExternalReference; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import java.util.ArrayList; @@ -93,12 +92,28 @@ private List processListsForProgram(List pro public List createBrAPILists(List brapiLists, UUID programId, ImportUpload upload) throws ApiException { ListsApi api = new ListsApi(programDAO.getCoreClient(programId)); // Do manually, it doesn't like List to List for some reason - ApiResponse response = api.listsPost(brapiLists); - if (response.getBody() == null) throw new ApiException("Response is missing body"); - BrAPIResponse body = (BrAPIResponse) response.getBody(); - if (body.getResult() == null) throw new ApiException("Response body is missing result"); - BrAPIResponseResult result = (BrAPIResponseResult) body.getResult(); - if (result.getData() == null) throw new ApiException("Response result is missing data"); - return result.getData(); + ApiResponse response; + try { + response = api.listsPost(brapiLists); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw e; + } + if(response != null) { + BrAPIResponse body = (BrAPIResponse) response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body"); + } + BrAPIResponseResult result = (BrAPIResponseResult) body.getResult(); + if (result == null) { + throw new ApiException("Response body is missing result"); + } + if (result.getData() == null) { + throw new ApiException("Response result is missing data"); + } + return result.getData(); + } + + throw new ApiException("No response after creating list"); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 8f52a0a13..64cc09978 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -730,6 +730,7 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { } } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); log.error(e.getResponseBody(), e);; } diff --git a/src/main/java/org/breedinginsight/daos/ObservationDAO.java b/src/main/java/org/breedinginsight/daos/ObservationDAO.java index 86f8ccc1e..9404937b5 100644 --- a/src/main/java/org/breedinginsight/daos/ObservationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ObservationDAO.java @@ -17,6 +17,7 @@ package org.breedinginsight.daos; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; @@ -29,14 +30,19 @@ import org.breedinginsight.services.brapi.BrAPIClientType; import org.breedinginsight.services.brapi.BrAPIProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Singleton; import java.util.List; import java.util.Optional; import java.util.UUID; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; + +@Singleton +@Slf4j public class ObservationDAO { private BrAPIProvider brAPIProvider; private final BrAPIDAOUtil brAPIDAOUtil; @@ -55,6 +61,7 @@ public List getObservationsByVariableDbId(String observationVa try { brapiObservations = brAPIProvider.getObservationsAPI(BrAPIClientType.PHENO).observationsGet(observationsRequest); } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); } diff --git a/src/main/java/org/breedinginsight/daos/ProgramDAO.java b/src/main/java/org/breedinginsight/daos/ProgramDAO.java index 0cf0845f6..3d3600da7 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramDAO.java @@ -17,350 +17,49 @@ package org.breedinginsight.daos; -import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Value; -import io.micronaut.http.server.exceptions.HttpServerException; -import io.micronaut.http.server.exceptions.InternalServerException; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.BrAPIClient; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams; -import org.brapi.client.v2.modules.core.ProgramsApi; -import org.brapi.client.v2.modules.core.ServerInfoApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.BrAPIWSMIMEDataTypes; import org.brapi.v2.model.core.BrAPIProgram; -import org.brapi.v2.model.core.response.BrAPIProgramListResponse; -import org.brapi.v2.model.core.response.BrAPIServerInfoResponse; -import org.breedinginsight.dao.db.tables.BiUserTable; -import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; -import org.breedinginsight.model.*; -import org.breedinginsight.model.User; -import org.breedinginsight.services.brapi.BrAPIClientProvider; -import org.breedinginsight.services.brapi.BrAPIClientType; -import org.breedinginsight.services.brapi.BrAPIProvider; -import org.jooq.*; -import org.jooq.tools.StringUtils; +import org.breedinginsight.dao.db.tables.records.ProgramRecord; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramBrAPIEndpoints; +import org.jooq.DAO; -import javax.inject.Inject; -import javax.inject.Singleton; -import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.UUID; -import static org.breedinginsight.dao.db.Tables.*; +public interface ProgramDAO extends DAO { + List get(List programIds); -@Slf4j -@Singleton -public class ProgramDAO extends ProgramDao { + List get(UUID programId); - @Property(name = "brapi.server.core-url") - private String defaultBrAPICoreUrl; - @Property(name = "brapi.server.pheno-url") - private String defaultBrAPIPhenoUrl; - @Property(name = "brapi.server.geno-url") - private String defaultBrAPIGenoUrl; + List getFromEntity(List programEntities); + List getAll(); - private DSLContext dsl; - private BrAPIProvider brAPIProvider; - private BrAPIClientProvider brAPIClientProvider; - @Property(name = "brapi.server.reference-source") - private String referenceSource; - private Duration requestTimeout; + List getActive(); - private final static String SYSTEM_DEFAULT = BrAPIConstants.SYSTEM_DEFAULT.getValue(); + List getProgramByName(String name, boolean caseInsensitive); - @Inject - public ProgramDAO(Configuration config, DSLContext dsl, BrAPIProvider brAPIProvider, BrAPIClientProvider brAPIClientProvider, - @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout) { - super(config); - this.dsl = dsl; - this.brAPIProvider = brAPIProvider; - this.brAPIClientProvider = brAPIClientProvider; - this.requestTimeout = requestTimeout; - } + List getProgramByKey(String key); - public List get(List programIds){ - return getPrograms(programIds); - } + int getNumProgramUsers(UUID programId); - public List get(UUID programId) { - List programList = new ArrayList<>(); - programList.add(programId); - return getPrograms(programList); - } + ProgramBrAPIEndpoints getProgramBrAPIEndpoints(UUID programId); - public List getFromEntity(List programEntities){ - List programList = new ArrayList<>(); - for (ProgramEntity programEntity: programEntities){ - programList.add(programEntity.getId()); - } - return getPrograms(programList); - } + ProgramEntity fetchOneById(UUID programId); - public List getAll() - { - return getPrograms(null); - } + List fetchById(UUID... values); - public List getProgramByName(String name, boolean caseInsensitive){ - BiUserTable createdByUser = BI_USER.as("createdByUser"); - BiUserTable updatedByUser = BI_USER.as("updatedByUser"); - Result queryResult; - List resultPrograms = new ArrayList<>(); + boolean brapiUrlSupported(String brapiUrl); - SelectOnConditionStep query = dsl.select() - .from(PROGRAM) - .join(SPECIES).on(PROGRAM.SPECIES_ID.eq(SPECIES.ID)) - .join(createdByUser).on(PROGRAM.CREATED_BY.eq(createdByUser.ID)) - .join(updatedByUser).on(PROGRAM.UPDATED_BY.eq(updatedByUser.ID)); + BrAPIProgram getProgramBrAPI(Program program); - if (caseInsensitive){ - queryResult = query - .where(PROGRAM.NAME.equalIgnoreCase(name)) - .fetch(); - } else { - queryResult = query - .where(PROGRAM.NAME.equal(name)) - .fetch(); - } + void createProgramBrAPI(Program program); - return parseProgramQuery(queryResult, createdByUser, updatedByUser); - } + void updateProgramBrAPI(Program program); - public List getProgramByKey(String key){ - BiUserTable createdByUser = BI_USER.as("createdByUser"); - BiUserTable updatedByUser = BI_USER.as("updatedByUser"); - Result queryResult; + BrAPIClient getCoreClient(UUID programId); - SelectOnConditionStep query = dsl.select() - .from(PROGRAM) - .join(SPECIES).on(PROGRAM.SPECIES_ID.eq(SPECIES.ID)) - .join(createdByUser).on(PROGRAM.CREATED_BY.eq(createdByUser.ID)) - .join(updatedByUser).on(PROGRAM.UPDATED_BY.eq(updatedByUser.ID)); - - queryResult = query - .where(PROGRAM.KEY.equal(key)) - .fetch(); - - return parseProgramQuery(queryResult, createdByUser, updatedByUser); - } - - private List getPrograms(List programIds){ - - BiUserTable createdByUser = BI_USER.as("createdByUser"); - BiUserTable updatedByUser = BI_USER.as("updatedByUser"); - Result queryResult; - List resultPrograms = new ArrayList<>(); - - SelectOnConditionStep query = dsl.select() - .from(PROGRAM) - .join(SPECIES).on(PROGRAM.SPECIES_ID.eq(SPECIES.ID)) - .join(createdByUser).on(PROGRAM.CREATED_BY.eq(createdByUser.ID)) - .join(updatedByUser).on(PROGRAM.UPDATED_BY.eq(updatedByUser.ID)); - - if (programIds != null){ - queryResult = query - .where(PROGRAM.ID.in(programIds)) - .fetch(); - } else { - queryResult = query.fetch(); - } - - return parseProgramQuery(queryResult, createdByUser, updatedByUser); - } - - private List parseProgramQuery(Result queryResult, BiUserTable createdByUser, BiUserTable updatedByUser) { - List resultPrograms = new ArrayList<>(); - for (Record record: queryResult){ - if (record.getValue(PROGRAM.BRAPI_URL) == null) { - record.setValue(PROGRAM.BRAPI_URL, SYSTEM_DEFAULT); - } - Program program = Program.parseSQLRecord(record); - // This will do some extra queries, performance may be better in combined query but was having some issues - // getting it working with jooq so went with this for now - program.setNumUsers(getNumProgramUsers(record.getValue(PROGRAM.ID))); - program.setSpecies(Species.parseSQLRecord(record)); - program.setCreatedByUser(User.parseSQLRecord(record, createdByUser)); - program.setUpdatedByUser(User.parseSQLRecord(record, updatedByUser)); - resultPrograms.add(program); - } - - return resultPrograms; - } - - public int getNumProgramUsers(UUID programId) { - return dsl.selectCount().from(PROGRAM_USER_ROLE) - .where(PROGRAM_USER_ROLE.PROGRAM_ID.eq(programId) - .and(PROGRAM_USER_ROLE.ACTIVE.eq(true))) - .fetchOne(0, Integer.class); - } - - public ProgramBrAPIEndpoints getProgramBrAPIEndpoints(UUID programId) { - ProgramEntity programEntity = fetchOneById(programId); - - String coreUrl = defaultBrAPICoreUrl; - String genoUrl = defaultBrAPIGenoUrl; - String phenoUrl = defaultBrAPIPhenoUrl; - - // only storing one program brapi url for now so set all to that one - if (!StringUtils.isBlank(programEntity.getBrapiUrl())) { - String brapiUrl = programEntity.getBrapiUrl(); - coreUrl = brapiUrl; - genoUrl = brapiUrl; - phenoUrl = brapiUrl; - } - - return ProgramBrAPIEndpoints.builder() - .coreUrl(Optional.of(coreUrl)) - .genoUrl(Optional.of(genoUrl)) - .phenoUrl(Optional.of(phenoUrl)) - .build(); - } - - public boolean brapiUrlSupported(String brapiUrl) { - boolean supported = true; - brAPIClientProvider.setBrapiClient(brapiUrl); - ServerInfoApi serverInfoAPI = brAPIProvider.getServerInfoAPI(BrAPIClientType.BRAPI); - - // for now just check for 200 response, in future we could check actual required endpoints - try { - ApiResponse response = serverInfoAPI.serverinfoGet(BrAPIWSMIMEDataTypes.APPLICATION_JSON); - } catch (ApiException e) { - log.error(e.getMessage()); - supported = false; - } - return supported; - } - - public BrAPIProgram getProgramBrAPI(Program program) { - - ProgramQueryParams searchRequest = new ProgramQueryParams() - .externalReferenceID(program.getId().toString()) - .externalReferenceSource(referenceSource); - - ProgramsApi programsApi = brAPIProvider.getProgramsAPI(BrAPIClientType.CORE); - // Get existing brapi program - ApiResponse brApiPrograms; - try { - brApiPrograms = programsApi.programsGet(searchRequest); - } catch (ApiException e) { - throw new HttpServerException("Could not find program in BrAPI service."); - } - - if (brApiPrograms.getBody().getResult().getData().isEmpty()) { - throw new HttpServerException("Could not find program in BrAPI service."); - } - - return brApiPrograms.getBody().getResult().getData().get(0); - } - - public void createProgramBrAPI(Program program) { - - BrAPIExternalReference externalReference = new BrAPIExternalReference() - .referenceID(program.getId().toString()) - .referenceSource(referenceSource); - - BrAPIProgram brApiProgram = new BrAPIProgram() - .programName(program.getName() + " (" + program.getKey() + ")") - .abbreviation(program.getAbbreviation()) - .commonCropName(program.getSpecies().getCommonName()) - .externalReferences(List.of(externalReference)) - .objective(program.getObjective()) - .documentationURL(program.getDocumentationUrl()); - - // POST programs to each brapi service - // TODO: If there is a failure after the first brapi service, roll back all before the failure. - try { - List programsAPIS = brAPIProvider.getAllUniqueProgramsAPI(); - for (ProgramsApi programsAPI: programsAPIS){ - programsAPI.programsPost(List.of(brApiProgram)); - } - } catch (ApiException e) { - log.debug(e.getMessage()); - log.debug(e.getResponseBody()); - log.debug(String.valueOf(e.getCode())); - throw new InternalServerException("Error making BrAPI call", e); - } - - } - - public void updateProgramBrAPI(Program program) { - - ProgramQueryParams searchRequest = new ProgramQueryParams() - .externalReferenceID(program.getId().toString()) - .externalReferenceSource(referenceSource); - - // Program goes in all of the clients - // TODO: If there is a failure after the first brapi service, roll back all before the failure. - List programsAPIS = brAPIProvider.getAllUniqueProgramsAPI(); - for (ProgramsApi programsAPI: programsAPIS){ - - // Get existing brapi program - ApiResponse brApiPrograms; - try { - brApiPrograms = programsAPI.programsGet(searchRequest); - } catch (ApiException e) { - throw new HttpServerException("Could not find program in BrAPI service."); - } - - if (brApiPrograms.getBody().getResult().getData().size() == 0){ - throw new HttpServerException("Could not find program in BrAPI service."); - } - - BrAPIProgram brApiProgram = brApiPrograms.getBody().getResult().getData().get(0); - - //TODO: Need to add archived/not archived when available in brapi - brApiProgram.setProgramName(program.getName() + " (" + program.getKey() + ")"); - brApiProgram.setAbbreviation(program.getAbbreviation()); - brApiProgram.setCommonCropName(program.getSpecies().getCommonName()); - brApiProgram.setObjective(program.getObjective()); - brApiProgram.setDocumentationURL(program.getDocumentationUrl()); - - try { - programsAPI.programsProgramDbIdPut(brApiProgram.getProgramDbId(), brApiProgram); - } catch (ApiException e) { - throw new HttpServerException("Could not find program in BrAPI service."); - } - } - } - - public BrAPIClient getCoreClient(UUID programId) { - Program program = get(programId).get(0); - String brapiUrl = !program.getBrapiUrl().equals(SYSTEM_DEFAULT) ? program.getBrapiUrl() : defaultBrAPICoreUrl; - BrAPIClient client = new BrAPIClient(brapiUrl); - initializeHttpClient(client); - return client; - } - - public BrAPIClient getPhenoClient(UUID programId) { - Program program = get(programId).get(0); - String brapiUrl = !program.getBrapiUrl().equals(SYSTEM_DEFAULT) ? program.getBrapiUrl() : defaultBrAPIPhenoUrl; - BrAPIClient client = new BrAPIClient(brapiUrl); - initializeHttpClient(client); - return client; - } - - private void initializeHttpClient(BrAPIClient brapiClient) { - brapiClient.setHttpClient(brapiClient.getHttpClient() - .newBuilder() - .readTimeout(getRequestTimeout()) - .build()); - } - - //TODO figure out why BrAPIServiceFilterIntegrationTest fails when requestTimeout is set in the constructor - private Duration getRequestTimeout() { - if(requestTimeout != null) { - return requestTimeout; - } - - return Duration.of(5, ChronoUnit.MINUTES); - } + BrAPIClient getPhenoClient(UUID programId); } - diff --git a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java index fc37b1633..d371c8103 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramLocationDAO.java @@ -24,6 +24,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.HttpServerException; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.model.queryParams.core.LocationQueryParams; @@ -36,6 +37,7 @@ import org.breedinginsight.dao.db.tables.daos.PlaceDao; import org.breedinginsight.model.*; import org.breedinginsight.services.brapi.BrAPIProvider; +import org.breedinginsight.utilities.Utilities; import org.jooq.Configuration; import org.jooq.DSLContext; import org.jooq.Record; @@ -51,6 +53,7 @@ import static org.breedinginsight.dao.db.Tables.*; @Singleton +@Slf4j public class ProgramLocationDAO extends PlaceDao { private DSLContext dsl; private BrAPIProvider brAPIProvider; @@ -169,6 +172,7 @@ public void createProgramLocationBrAPI(ProgramLocation location) { locationsAPI.locationsPost(List.of(brApiLocation)); } } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new InternalServerException("Error making BrAPI call", e); } @@ -190,6 +194,7 @@ public void updateProgramLocationBrAPI(ProgramLocation location) { try { brApiLocations = locationsAPI.locationsGet(searchRequest); } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new HttpServerException("Could not find location in BrAPI service."); } @@ -221,6 +226,7 @@ public void updateProgramLocationBrAPI(ProgramLocation location) { try { locationsAPI.locationsLocationDbIdPut(brApiLocation.getLocationDbId(), brApiLocation); } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); throw new HttpServerException("Could not find location in BrAPI service."); } } diff --git a/src/main/java/org/breedinginsight/daos/TraitDAO.java b/src/main/java/org/breedinginsight/daos/TraitDAO.java index f929bdffe..c0cd1b5b7 100644 --- a/src/main/java/org/breedinginsight/daos/TraitDAO.java +++ b/src/main/java/org/breedinginsight/daos/TraitDAO.java @@ -1,603 +1,52 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.daos; -import com.google.gson.Gson; -import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; -import org.brapi.client.v2.ApiResponse; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.phenotype.VariableQueryParams; -import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.pheno.*; -import org.brapi.v2.model.pheno.request.BrAPIObservationVariableSearchRequest; -import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; -import org.breedinginsight.dao.db.tables.BiUserTable; -import org.breedinginsight.dao.db.tables.daos.TraitDao; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationVariable; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.dao.db.tables.records.TraitRecord; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; -import org.breedinginsight.model.*; -import org.breedinginsight.services.brapi.BrAPIProvider; -import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.jooq.*; -import org.jooq.tools.StringUtils; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.breedinginsight.dao.db.Tables.*; -import static org.breedinginsight.services.brapi.BrAPIClientType.PHENO; -import static org.jooq.impl.DSL.*; - -@Singleton -public class TraitDAO extends TraitDao { - - private DSLContext dsl; - private BrAPIProvider brAPIProvider; - @Property(name = "brapi.server.reference-source") - private String referenceSource; - private ObservationDAO observationDao; - private final BrAPIDAOUtil brAPIDAOUtil; - private Gson gson; - - private final static String TAGS_KEY = "tags"; - private final static String FULLNAME_KEY = "fullname"; - - @Inject - public TraitDAO(Configuration config, DSLContext dsl, BrAPIProvider brAPIProvider, ObservationDAO observationDao, BrAPIDAOUtil brAPIDAOUtil) { - super(config); - this.dsl = dsl; - this.brAPIProvider = brAPIProvider; - this.observationDao = observationDao; - this.brAPIDAOUtil = brAPIDAOUtil; - this.gson = new Gson(); - } - - public List getTraitsFullByProgramId(UUID programId) { - List programIds = new ArrayList<>(); - programIds.add(programId); - return getTraitsFullByProgramIds(programIds); - } - - public List getTraitsFullByProgramIds(List programIds) { - - // Get our db traits (equivalent to brapi variables) - List dbVariables = getTraitsByProgramIds(programIds.toArray(UUID[]::new)); - if (dbVariables.size() == 0){ - return new ArrayList<>(); - } - Map dbVariablesMap = dbVariables.stream().collect(Collectors.toMap(Trait::getId, p -> p)); - - // Get brapi variables - VariableQueryParams variablesRequest = new VariableQueryParams(); - variablesRequest.externalReferenceSource(referenceSource); - variablesRequest.pageSize(10000); - - ApiResponse brApiVariables; - try { - brApiVariables = brAPIProvider.getVariablesAPI(PHENO).variablesGet(variablesRequest); - } catch (ApiException e) { - throw new InternalServerException("Error making BrAPI call", e); - } - - Map brApiVariableMap = new HashMap<>(); - for (BrAPIObservationVariable brApiVariable: brApiVariables.getBody().getResult().getData()) { - List brApiExternalReferences = brApiVariable.getExternalReferences(); - for (BrAPIExternalReference brApiExternalReference: brApiExternalReferences){ - if (brApiExternalReference.getReferenceID() != null) { - brApiVariableMap.put(brApiExternalReference.getReferenceID(), brApiVariable); - } - } - } - - List saturatedTraits = new ArrayList<>(); - for (Trait trait: dbVariables) { - // assumes external reference id is unique to each brapi variable - if (brApiVariableMap.containsKey(trait.getId().toString())){ - BrAPIObservationVariable brApiVariable = brApiVariableMap.get(trait.getId().toString()); - saturateTrait(trait, brApiVariable); - saturatedTraits.add(trait); - } else { - throw new InternalServerException("Could not find trait in returned brapi server results"); - } - } - - return saturatedTraits; - } - - public List getTraitsByProgramId(UUID programId) { - return getTraitsByProgramIds(programId); - } - - public List getTraitsByProgramIds(UUID ...programIds) { - - BiUserTable createdByUser = BI_USER.as("createdByUser"); - BiUserTable updatedByUser = BI_USER.as("updatedByUser"); - - Result recordResult = getTraitSql(createdByUser, updatedByUser) - .where(PROGRAM_ONTOLOGY.PROGRAM_ID.in(programIds)) - .fetch(); - - List traitResults = new ArrayList<>(); - for (Record record: recordResult) { - Trait trait = parseTraitRecord(record, createdByUser, updatedByUser); - traitResults.add(trait); - } - - return traitResults; - } - - public List getTraitsById(UUID ...traitIds){ +import org.jooq.DAO; - BiUserTable createdByUser = BI_USER.as("createdByUser"); - BiUserTable updatedByUser = BI_USER.as("updatedByUser"); +import java.util.List; +import java.util.Optional; +import java.util.UUID; - Result recordResult = getTraitSql(createdByUser, updatedByUser) - .and(TRAIT.ID.in(traitIds)) - .fetch(); +public interface TraitDAO extends DAO { + List getTraitsFullByProgramId(UUID programId); - List traitResults = new ArrayList<>(); - for (Record record: recordResult) { - Trait trait = parseTraitRecord(record, createdByUser, updatedByUser); - traitResults.add(trait); - } + List getTraitsFullByProgramIds(List programIds); - return traitResults; - } + List getTraitsByProgramId(UUID programId); - public Optional getTraitFull(UUID programId, UUID traitId){ + List getTraitsByProgramIds(UUID... programIds); - Optional optionalDbTrait = getTrait(programId, traitId); - if (!optionalDbTrait.isPresent()){ - return Optional.empty(); - } - Trait dbTrait = optionalDbTrait.get(); + List getTraitsById(UUID... traitIds); - ApiResponse brApiVariables; - - VariableQueryParams variablesRequest = new VariableQueryParams() - .externalReferenceID(traitId.toString()) - .externalReferenceSource(referenceSource); - try { - brApiVariables = brAPIProvider.getVariablesAPI(PHENO).variablesGet(variablesRequest); - } catch (ApiException e) { - // If variable is not found, is still a server exception - throw new InternalServerException("Error making BrAPI call", e); - } - - BrAPIObservationVariable brApiVariable; - if (brApiVariables.getBody().getResult().getData().size() > 0){ - brApiVariable = brApiVariables.getBody().getResult().getData().get(0); - } else { - throw new InternalServerException("No variable found in brapi server"); - } - - saturateTrait(dbTrait, brApiVariable); - return Optional.of(dbTrait); - } + Optional getTraitFull(UUID programId, UUID traitId); // could be more efficient to do a single get instead of search in saved search case but less code this way // and search stuff is working in breedbase - public List getObservationsForTrait(UUID traitId) { - return getObservationsForTraits(Stream.of(traitId).collect(Collectors.toList())); - } - - public List getObservationsForTraits(List traitIds) { - - List ids = traitIds.stream() - .map(trait -> trait.toString()) - .collect(Collectors.toList()); - - List variables = searchVariables(ids); - - // TODO: make sure have all expected external references - if (variables.size() != ids.size()) { - throw new InternalServerException("Observation variables search results mismatch"); - } - - List brapiVariableIds = variables.stream() - .map(variable -> variable.getObservationVariableDbId()).collect(Collectors.toList()); - - return observationDao.getObservationsByVariableDbIds(brapiVariableIds); - } - - public List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, List traitIds) { - - List ids = traitIds.stream() - .map(trait -> trait.toString()) - .collect(Collectors.toList()); - - List variables = searchVariables(ids); - - // TODO: make sure have all expected external references - if (variables.size() != ids.size()) { - throw new InternalServerException("Observation variables search results mismatch"); - } - - List brapiVariableIds = variables.stream() - .map(variable -> variable.getObservationVariableDbId()).collect(Collectors.toList()); - - return observationDao.getObservationsByVariableAndBrAPIProgram(brapiProgramId, brapiVariableIds); - } - - public List searchVariables(List variableIds) { - - if (variableIds == null || variableIds.size() == 0) return new ArrayList<>(); - try { - BrAPIObservationVariableSearchRequest request = new BrAPIObservationVariableSearchRequest() - .externalReferenceIDs(variableIds); - - ObservationVariablesApi api = brAPIProvider.getVariablesAPI(PHENO); - return brAPIDAOUtil.search( - api::searchVariablesPost, - api::searchVariablesSearchResultsDbIdGet, - request - ); - } catch (ApiException e) { - throw new InternalServerException("Observation variables brapi search error", e); - } - } - - public Optional getTrait(UUID programId, UUID traitId) { - - BiUserTable createdByUser = BI_USER.as("createdByUser"); - BiUserTable updatedByUser = BI_USER.as("updatedByUser"); - - Record record = getTraitSql(createdByUser, updatedByUser) - .where(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(programId)) - .and(TRAIT.ID.eq(traitId)) - .fetchOne(); - - if (record == null) { - return Optional.empty(); - } - - return Optional.of(parseTraitRecord(record, createdByUser, updatedByUser)); - } - - public List createTraitsBrAPI(List traits, User actingUser, Program program){ - - //TODO: Pass ontology reference - - // Convert our traits into BrAPI traits - List brApiVariables = new ArrayList<>(); - for (Trait trait: traits) { - - // Construct method - BrAPIExternalReference methodReference = new BrAPIExternalReference() - .referenceID(trait.getMethod().getId().toString()) - .referenceSource(referenceSource); - BrAPIMethod brApiMethod = new BrAPIMethod() - .methodName(constructMethodName(trait, program)) - .externalReferences(List.of(methodReference)) - .methodClass(trait.getMethod().getMethodClass()) - .description(trait.getMethod().getDescription()) - .formula(trait.getMethod().getFormula()); - - // Construct scale - BrAPIExternalReference scaleReference = new BrAPIExternalReference() - .referenceID(trait.getScale().getId().toString()) - .referenceSource(referenceSource); - BrAPITraitDataType brApiTraitDataType = BrAPITraitDataType.valueOf(trait.getScale().getDataType().toString()); - BrAPIScaleValidValues brApiScaleValidValues = new BrAPIScaleValidValues() - .categories(trait.getScale().getCategories()) - .max(trait.getScale().getValidValueMax()) - .min(trait.getScale().getValidValueMin()); - BrAPIScale brApiScale = new BrAPIScale() - .scaleName(String.format("%s [%s]", trait.getScale().getScaleName(), program.getKey())) - .externalReferences(List.of(scaleReference)) - .dataType(brApiTraitDataType) - .decimalPlaces(trait.getScale().getDecimalPlaces()) - .validValues(brApiScaleValidValues); - - // Construct trait - BrAPIExternalReference traitReference = new BrAPIExternalReference() - .referenceID(trait.getId().toString()) - .referenceSource(referenceSource); - BrAPITrait brApiTrait = new BrAPITrait() - .traitName(String.format("%s %s [%s]", trait.getEntity(), trait.getAttribute(), program.getKey())) - .traitDescription(trait.getTraitDescription()) - .synonyms(trait.getSynonyms()) - .status("active") - .entity(trait.getEntity()) - .mainAbbreviation(trait.getMainAbbreviation()) - .traitClass(trait.getTraitClass()) - .externalReferences(List.of(traitReference)) - .attribute(trait.getAttribute()); - - BrAPIExternalReference variableReference = new BrAPIExternalReference() - .referenceID(trait.getId().toString()) - .referenceSource(referenceSource); - BrAPIObservationVariable brApiVariable = new BrAPIObservationVariable() - .method(brApiMethod) - .scale(brApiScale) - .trait(brApiTrait) - .externalReferences(List.of(variableReference)) - .observationVariableName(String.format("%s [%s]", trait.getObservationVariableName(), program.getKey())) - .status("active") - .language("english") - .scientist(actingUser.getName()) - .defaultValue(trait.getDefaultValue()) - .synonyms(trait.getSynonyms()) - .institution(program.getName()) - .commonCropName(program.getSpecies().getCommonName()); - if (trait.getTags() != null) brApiVariable.putAdditionalInfoItem(TAGS_KEY, trait.getTags()); - if (trait.getFullName() != null) brApiVariable.putAdditionalInfoItem(FULLNAME_KEY, trait.getFullName()); - - if (trait.getActive() == null || trait.getActive()){ - brApiVariable.setStatus("active"); - } else { - brApiVariable.setStatus("archived"); - } - - - // Unused - //.contextOfUse() - //.documentationURL() - //.growthStage() - - brApiVariables.add(brApiVariable); - } - - - // POST variables to each brapi service - // TODO: If there is a failure after the first brapi service, roll back all before the failure. - ApiResponse createdVariables = null; - try { - List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); - for (ObservationVariablesApi variablesAPI: variablesAPIS){ - createdVariables = variablesAPI.variablesPost(brApiVariables); - } - } catch (ApiException e) { - throw new InternalServerException("Error making BrAPI call", e); - } - - if(createdVariables == null) { - throw new InternalServerException("Creating new variable did not return any data"); - } - - // Pull our traits from the db - List traitIds = traits.stream().map(trait -> trait.getId()).collect(Collectors.toList()); - List createdTraits = getTraitsById(traitIds.toArray(UUID[]::new)); - - // Saturate our traits from the brapi return information - for (Trait trait: createdTraits){ - for (BrAPIObservationVariable variable: createdVariables.getBody().getResult().getData()){ - if (variable.getExternalReferences() != null) { - for (BrAPIExternalReference brApiExternalReference: variable.getExternalReferences()){ - if (brApiExternalReference.getReferenceSource().equals(referenceSource) && - brApiExternalReference.getReferenceID().equals(trait.getId().toString())){ - - saturateTrait(trait, variable); - } - } - } - } - } - - return createdTraits; - } - - public Trait updateTraitBrAPI(Trait trait, Program program) { - - //TODO: Need to roll back somehow if there is an error - Trait updatedTrait = null; - List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); - for (ObservationVariablesApi variablesAPI: variablesAPIS){ - // GET brapi trait - BrAPIObservationVariable existingVariable = getBrAPIVariable(variablesAPI, trait.getId()); - - // Change method - existingVariable.getMethod().setMethodName(constructMethodName(trait, program)); - existingVariable.getMethod().setMethodClass(trait.getMethod().getMethodClass()); - existingVariable.getMethod().setDescription(trait.getMethod().getDescription()); - existingVariable.getMethod().setFormula(trait.getMethod().getFormula()); - - // Change scale - BrAPITraitDataType brApiTraitDataType = BrAPITraitDataType.valueOf(trait.getScale().getDataType().toString()); - existingVariable.getScale().setScaleName(String.format("%s [%s]", trait.getScale().getScaleName(), program.getKey())); - existingVariable.getScale().setDataType(brApiTraitDataType); - existingVariable.getScale().setDecimalPlaces(trait.getScale().getDecimalPlaces()); - BrAPIScaleValidValues brApiScaleValidValues = new BrAPIScaleValidValues() - .categories(trait.getScale().getCategories()) - .max(trait.getScale().getValidValueMax()) - .min(trait.getScale().getValidValueMin()); - existingVariable.getScale().setValidValues(brApiScaleValidValues); - - // Change trait - existingVariable.getTrait().setTraitName(String.format("%s %s [%s]", trait.getEntity(), trait.getAttribute(), program.getKey())); - existingVariable.getTrait().setTraitDescription(trait.getTraitDescription()); - existingVariable.getTrait().setSynonyms(trait.getSynonyms()); - existingVariable.getTrait().setEntity(trait.getProgramObservationLevel().getName()); - existingVariable.getTrait().setMainAbbreviation(trait.getMainAbbreviation()); - existingVariable.getTrait().setTraitClass(trait.getTraitClass()); - existingVariable.getTrait().setAttribute(trait.getAttribute()); - - // Change variable - existingVariable.setObservationVariableName(String.format("%s [%s]", trait.getObservationVariableName(), program.getKey())); - existingVariable.setDefaultValue(trait.getDefaultValue()); - existingVariable.setSynonyms(trait.getSynonyms()); - if (trait.getActive() == null || trait.getActive()){ - existingVariable.setStatus("active"); - } else { - existingVariable.setStatus("archived"); - } - existingVariable.putAdditionalInfoItem(TAGS_KEY, trait.getTags()); - if (trait.getFullName() != null) existingVariable.putAdditionalInfoItem(FULLNAME_KEY, trait.getFullName()); - - // PUT brapi trait - BrAPIObservationVariable updatedVariable = putBrAPIVariable(variablesAPI, existingVariable); - - // Retrieve our update trait from the db - updatedTrait = getTrait(program.getId(), trait.getId()).get(); - saturateTrait(updatedTrait, updatedVariable); - } - - return updatedTrait; - } - - private String constructMethodName(Trait trait, Program program) { - return !StringUtils.isBlank(trait.getMethod().getDescription()) ? - String.format("%s %s [%s]", trait.getMethod().getDescription(), trait.getMethod().getMethodClass(), program.getKey()) : - String.format("%s [%s]", trait.getMethod().getMethodClass(), program.getKey()); - } - - private BrAPIObservationVariable getBrAPIVariable(ObservationVariablesApi variablesAPI, UUID traitId) { - - BrAPIObservationVariable existingVariable; - try { - VariableQueryParams queryParams = new VariableQueryParams(); - queryParams.externalReferenceID(traitId.toString()); - queryParams.externalReferenceSource(referenceSource); - ApiResponse existingVariableResponse = - variablesAPI.variablesGet(queryParams); - List variableList = existingVariableResponse.getBody().getResult().getData(); - if (variableList.size() == 1) { - existingVariable = variableList.get(0); - } else { - throw new InternalServerException(String.format("Unable to find variable with id %s in brapi server.", traitId.toString())); - } - - } catch (ApiException e) { - throw new InternalServerException(String.format("Unable to retrieve variable with id %s in brapi server.", traitId.toString())); - } - - return existingVariable; - } - - private BrAPIObservationVariable putBrAPIVariable(ObservationVariablesApi variablesAPI, BrAPIObservationVariable variable) { - - BrAPIObservationVariable updatedVariable; - try { - ApiResponse updatedResponse = - variablesAPI.variablesObservationVariableDbIdPut(variable.getObservationVariableDbId(), variable); - updatedVariable = updatedResponse.getBody().getResult(); - } catch (ApiException e) { - throw new InternalServerException(String.format("Unable to save variable in brapi server.")); - } - return updatedVariable; - } - - private Trait parseTraitRecord(Record record, BiUserTable createdByUser, BiUserTable updatedByUser) { - Trait trait = Trait.parseSqlRecord(record); - Scale scale = Scale.parseSqlRecord(record); - Method method = Method.parseSqlRecord(record); - ProgramOntology programOntology = ProgramOntology.parseSqlRecord(record); - ProgramObservationLevel programObservationLevel = ProgramObservationLevel.parseSqlRecord(record); - User createUser = User.parseSQLRecord(record, createdByUser); - User updateUser = User.parseSQLRecord(record, updatedByUser); - - trait.setScale(scale); - trait.setMethod(method); - trait.setProgramOntology(programOntology); - trait.setProgramObservationLevel(programObservationLevel); - trait.setCreatedByUser(createUser); - trait.setUpdatedByUser(updateUser); - - return trait; - } - - private SelectOnConditionStep getTraitSql(BiUserTable createdByTableAlias, BiUserTable updatedByTableAlias) { - return dsl.select() - .from(TRAIT) - .join(METHOD).on(TRAIT.METHOD_ID.eq(METHOD.ID)) - .join(SCALE).on(TRAIT.SCALE_ID.eq(SCALE.ID)) - .join(PROGRAM_OBSERVATION_LEVEL).on(TRAIT.PROGRAM_OBSERVATION_LEVEL_ID.eq(PROGRAM_OBSERVATION_LEVEL.ID)) - .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) - .join(createdByTableAlias).on(TRAIT.CREATED_BY.eq(createdByTableAlias.ID)) - .join(updatedByTableAlias).on(TRAIT.UPDATED_BY.eq(updatedByTableAlias.ID)); - } - - public List getTraitsByTraitName(UUID programId, List traits){ - - String[] names = traits.stream() - .filter(trait -> trait.getObservationVariableName() != null) - .map(trait -> trait.getObservationVariableName().toLowerCase()) - .collect(Collectors.toList()).toArray(String[]::new); - - List traitResults = new ArrayList<>(); - if (names.length > 0){ - - Result records = dsl.select() - .from(TRAIT) - .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) - .join(PROGRAM).on(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(PROGRAM.ID)) - .join(SCALE).on(TRAIT.SCALE_ID.eq(SCALE.ID)) - .join(METHOD).on(TRAIT.METHOD_ID.eq(METHOD.ID)) - .where(PROGRAM.ID.eq(programId)) - .and(lower(TRAIT.OBSERVATION_VARIABLE_NAME).in(names)) - .fetch(); + List getObservationsForTrait(UUID traitId); - for (Record record: records) { - Trait trait = Trait.parseSqlRecord(record); - Scale scale = Scale.parseSqlRecord(record); - Method method = Method.parseSqlRecord(record); - trait.setScale(scale); - trait.setMethod(method); - traitResults.add(trait); - } - } + List getObservationsForTraits(List traitIds); - return traitResults; - } + List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, List traitIds); - public List getTraitsByAbbreviation(UUID programId, List abbreviations) { + List searchVariables(List variableIds); - Result records = dsl.select() - .from(TRAIT) - .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) - .join(PROGRAM).on(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(PROGRAM.ID)) - .and(PROGRAM.ID.eq(programId)) - .fetch(); + Optional getTrait(UUID programId, UUID traitId); - List traitResults = new ArrayList<>(); - for (Record record: records) { - Trait trait = Trait.parseSqlRecord(record); - traitResults.add(trait); - } + List createTraitsBrAPI(List traits, User actingUser, Program program); - return traitResults; - } + Trait updateTraitBrAPI(Trait trait, Program program); - private void saturateTrait(Trait trait, BrAPIObservationVariable brApiVariable) { + List getTraitsByTraitName(UUID programId, List traits); - if (brApiVariable.getAdditionalInfo() != null) { - List tags = null; - String fullName = null; - if (brApiVariable.getAdditionalInfo().has(TAGS_KEY) && !brApiVariable.getAdditionalInfo().get(TAGS_KEY).isJsonNull()) { - tags = gson.fromJson(brApiVariable.getAdditionalInfo().getAsJsonArray(TAGS_KEY), List.class); - } - if (brApiVariable.getAdditionalInfo().has(FULLNAME_KEY) && !brApiVariable.getAdditionalInfo().get(FULLNAME_KEY).isJsonNull()) { - fullName = brApiVariable.getAdditionalInfo().get(FULLNAME_KEY).getAsString(); - } - trait.setBrAPIProperties(brApiVariable, tags, fullName); - } else { - trait.setBrAPIProperties(brApiVariable); - } + List getTraitsByAbbreviation(UUID programId, List abbreviations); - Method method = trait.getMethod(); - method.setBrAPIProperties(brApiVariable.getMethod()); + TraitEntity fetchOneById(UUID id); - Scale scale = trait.getScale(); - scale.setBrAPIProperties(brApiVariable.getScale()); - } + List fetchById(UUID traitId); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/FetchFunction.java b/src/main/java/org/breedinginsight/daos/cache/FetchFunction.java similarity index 79% rename from src/main/java/org/breedinginsight/brapi/v2/dao/FetchFunction.java rename to src/main/java/org/breedinginsight/daos/cache/FetchFunction.java index 3df92e01d..d9e1dc992 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/FetchFunction.java +++ b/src/main/java/org/breedinginsight/daos/cache/FetchFunction.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapi.v2.dao; +package org.breedinginsight.daos.cache; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java new file mode 100644 index 000000000..6dd2f78aa --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -0,0 +1,190 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.daos.cache; + +import com.google.gson.Gson; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.JSON; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.redisson.api.*; + +import javax.validation.constraints.NotNull; + +import java.util.*; +import java.util.concurrent.*; + +/** + * + * @param object + */ +@Slf4j +public class ProgramCache { + private final RedissonClient connection; + private final Gson gson; + private final FetchFunction> fetchMethod; + private final Map programSemaphore = new HashMap<>(); + private final Map programQueueSemaphore = new HashMap<>(); + private Class type; + private final Executor executor = Executors.newCachedThreadPool(); + + public ProgramCache(RedissonClient connection, FetchFunction> fetchMethod, Class type) { + this.connection = connection; + this.gson = new JSON().getGson(); + this.fetchMethod = fetchMethod; + this.type = type; + } + + public void populate(List keys) { + for(UUID key : keys) { + populate(key); + } + } + + public void populate(@NotNull UUID key) { + String cacheKey = generateCacheKey(key); + if (!programSemaphore.containsKey(cacheKey)) { + RSemaphore semaphore = connection.getSemaphore(cacheKey+":semaphore"); + semaphore.trySetPermits(1); + programSemaphore.put(cacheKey, semaphore); + + RSemaphore queueSemaphore = connection.getSemaphore(cacheKey+":semaphore:queue"); + queueSemaphore.trySetPermits(1); + programQueueSemaphore.put(cacheKey, queueSemaphore); + } + + boolean acquired = programSemaphore.get(cacheKey).tryAcquire(); + + boolean refresh = true; + if(!acquired) { + /* + put this thread in line to refresh once the current refresh finishes. + If there is already a thread in line, let this thread finish as the + next refresh will pick up data persisted by this thread + */ + if(programQueueSemaphore.get(cacheKey).tryAcquire()) { + try { + // block until we get the green light to refresh the cache + programSemaphore.get(cacheKey).acquire(); + // and let go of our hold on the refresh queue + programQueueSemaphore.get(cacheKey).release(); + log.debug("repopulating cache for " + cacheKey); + } catch (InterruptedException e) { + log.error("Error acquiring lock to refresh "+cacheKey, e); + throw new RuntimeException(e); + } + } else { + log.debug("A refresh is queued up for key: "+cacheKey+", leaving"); + refresh = false; + } + } + + if (refresh) { // if false, an update is already in progress and another one queued up, skip this one + // Start a refresh process asynchronously + executor.execute(() -> { + // Synchronous + try { + log.debug("loading cache for key: " + cacheKey); + connection.getAtomicLong(cacheKey+":refreshing").set(1); + Map values = fetchMethod.apply(key); + if(!values.isEmpty()) { + log.debug("Caching new values for key: " + cacheKey); + Map entryMap = new HashMap<>(); + for (Map.Entry val : values.entrySet()) { + String entryVal = gson.toJson(val.getValue()); + entryMap.put(val.getKey(), entryVal); + } + + RMap map = connection.getMap(cacheKey); + map.clear(); + map.putAll(entryMap); + } + log.debug("cache loading complete for key: " + cacheKey); + } catch (Exception e) { + log.error("cache loading error for key: " + cacheKey, e); + invalidate(key); + throw new InternalServerException(e.getMessage(), e); + } finally { + connection.getAtomicLong(cacheKey+":refreshing").set(0); + programSemaphore.get(cacheKey).release(); + } + }); + } + } + + public void set(@NotNull UUID key, @NotNull String id, @NotNull R value) { + connection.getMap(generateCacheKey(key)).put(id, gson.toJson(value)); + } + + public void invalidate(@NotNull UUID key) { + connection.getMap(generateCacheKey(key)).delete(); + } + + public Map get(UUID key) throws ApiException { + String cacheKey = generateCacheKey(key); + log.debug("Getting for key: " + cacheKey); + + try { + if (!connection.getBucket(cacheKey).isExists()) { + log.debug("cache miss, populating"); + populate(key); + //block until any updates are done + programSemaphore.get(cacheKey).acquire(); + programSemaphore.get(cacheKey).release(); + } + return deserialize(connection.getMap(cacheKey)); + } catch (Exception e) { + throw new ApiException(e); + } + } + + private Map deserialize(Map cachedVals) { + Map retMap = new HashMap<>(); + cachedVals.forEach((key, value) -> retMap.put(key, gson.fromJson(value, type))); + return retMap; + } + + public List post(UUID key, Callable> postMethod) { + log.debug("posting for key: " + generateCacheKey(key)); + Map response = null; + try { + response = postMethod.call(); + + String cacheKey = generateCacheKey(key); + RMap map = connection.getMap(cacheKey); + //temporarily populate the cache with the returned objects from the postMethod so they show in immediate cache requests + for(Map.Entry obj : response.entrySet()) { + map.put(obj.getKey(), gson.toJson(obj.getValue())); + } + populate(key); + } catch (Exception e) { + invalidate(key); + } + return new ArrayList<>(response.values()); + } + + public boolean isRefreshing(UUID key) { + RAtomicLong isRefreshing = connection.getAtomicLong(generateCacheKey(key) + ":refreshing"); + + return isRefreshing.get() == 1; + } + + private String generateCacheKey(UUID key) { + return key.toString() + ":" + type.getSimpleName().toLowerCase(); + } +} diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCacheProvider.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCacheProvider.java new file mode 100644 index 000000000..321aeb041 --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCacheProvider.java @@ -0,0 +1,22 @@ +package org.breedinginsight.daos.cache; + +import org.redisson.api.RedissonClient; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Map; +import java.util.UUID; + +@Singleton +public class ProgramCacheProvider { + private final RedissonClient connection; + + @Inject + public ProgramCacheProvider(RedissonClient connection) { + this.connection = connection; + } + + public ProgramCache getProgramCache(FetchFunction> fetchMethod, Class type) { + return new ProgramCache<>(connection, fetchMethod, type); + } +} diff --git a/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java b/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java new file mode 100644 index 000000000..138cf73c4 --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java @@ -0,0 +1,187 @@ +package org.breedinginsight.daos.impl; + +import lombok.Getter; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jooq.*; +import org.jooq.conf.Settings; +import org.jooq.exception.DataAccessException; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +public abstract class AbstractDAO, P, T> implements DAO { + + @Getter + private final DAO jooqDAO; + + public AbstractDAO(DAO jooqDAO) { + this.jooqDAO = jooqDAO; + } + + @Override + public @NotNull Configuration configuration() { + return jooqDAO.configuration(); + } + + @Override + public @NotNull Settings settings() { + return jooqDAO.settings(); + } + + @Override + public @NotNull SQLDialect dialect() { + return jooqDAO.dialect(); + } + + @Override + public @NotNull SQLDialect family() { + return jooqDAO.family(); + } + + @Override + public @NotNull RecordMapper mapper() { + return jooqDAO.mapper(); + } + + @Override + public void insert(P object) throws DataAccessException { + jooqDAO.insert(object); + } + + @Override + public void insert(P... objects) throws DataAccessException { + jooqDAO.insert(objects); + } + + @Override + public void insert(Collection

objects) throws DataAccessException { + jooqDAO.insert(objects); + } + + @Override + public void update(P object) throws DataAccessException { + jooqDAO.update(object); + } + + @Override + public void update(P... objects) throws DataAccessException { + jooqDAO.update(objects); + } + + @Override + public void update(Collection

objects) throws DataAccessException { + jooqDAO.update(objects); + } + + @Override + public void merge(P object) throws DataAccessException { + jooqDAO.merge(object); + } + + @Override + public void merge(P... objects) throws DataAccessException { + jooqDAO.merge(objects); + } + + @Override + public void merge(Collection

objects) throws DataAccessException { + jooqDAO.merge(objects); + } + + @Override + public void delete(P object) throws DataAccessException { + jooqDAO.delete(object); + } + + @Override + public void delete(P... objects) throws DataAccessException { + jooqDAO.delete(objects); + } + + @Override + public void delete(Collection

objects) throws DataAccessException { + jooqDAO.delete(objects); + } + + @Override + public void deleteById(T... ids) throws DataAccessException { + jooqDAO.deleteById(ids); + } + + @Override + public void deleteById(Collection ids) throws DataAccessException { + jooqDAO.deleteById(ids); + } + + @Override + public boolean exists(P object) throws DataAccessException { + return jooqDAO.exists(object); + } + + @Override + public boolean existsById(T id) throws DataAccessException { + return jooqDAO.existsById(id); + } + + @Override + public long count() throws DataAccessException { + return jooqDAO.count(); + } + + @Override + public @NotNull List

findAll() throws DataAccessException { + return jooqDAO.findAll(); + } + + @Override + public @Nullable P findById(T id) throws DataAccessException { + return jooqDAO.findById(id); + } + + @Override + public @NotNull Optional

findOptionalById(T id) throws DataAccessException { + return jooqDAO.findOptionalById(id); + } + + @Override + public @NotNull List

fetch(Field field, Z... values) throws DataAccessException { + return jooqDAO.fetch(field, values); + } + + @Override + public @NotNull List

fetch(Field field, Collection values) throws DataAccessException { + return jooqDAO.fetch(field, values); + } + + @Override + public @NotNull List

fetchRange(Field field, Z lowerInclusive, Z upperInclusive) throws DataAccessException { + return jooqDAO.fetchRange(field, lowerInclusive, upperInclusive); + } + + @Override + public @Nullable P fetchOne(Field field, Z value) throws DataAccessException { + return jooqDAO.fetchOne(field, value); + } + + @Override + public @NotNull Optional

fetchOptional(Field field, Z value) throws DataAccessException { + return jooqDAO.fetchOptional(field, value); + } + + @Override + public @NotNull Table getTable() { + return jooqDAO.getTable(); + } + + @Override + public @NotNull Class

getType() { + return jooqDAO.getType(); + } + + @Override + public T getId(P object) { + return jooqDAO.getId(object); + } +} diff --git a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java new file mode 100644 index 000000000..ae3b6f699 --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java @@ -0,0 +1,405 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.daos.impl; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Value; +import io.micronaut.http.server.exceptions.HttpServerException; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.core.ProgramQueryParams; +import org.brapi.client.v2.modules.core.ProgramsApi; +import org.brapi.client.v2.modules.core.ServerInfoApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIWSMIMEDataTypes; +import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.core.response.BrAPIProgramListResponse; +import org.brapi.v2.model.core.response.BrAPIServerInfoResponse; +import org.breedinginsight.dao.db.tables.BiUserTable; +import org.breedinginsight.dao.db.tables.daos.ProgramDao; +import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; +import org.breedinginsight.dao.db.tables.records.ProgramRecord; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.User; +import org.breedinginsight.model.*; +import org.breedinginsight.services.brapi.BrAPIClientProvider; +import org.breedinginsight.services.brapi.BrAPIClientType; +import org.breedinginsight.services.brapi.BrAPIProvider; +import org.breedinginsight.utilities.Utilities; +import org.jooq.*; +import org.jooq.tools.StringUtils; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.*; + +@Slf4j +@Singleton +public class ProgramDAOImpl extends AbstractDAO implements ProgramDAO { + + @Property(name = "brapi.server.core-url") + private String defaultBrAPICoreUrl; + @Property(name = "brapi.server.pheno-url") + private String defaultBrAPIPhenoUrl; + @Property(name = "brapi.server.geno-url") + private String defaultBrAPIGenoUrl; + + + private DSLContext dsl; + private BrAPIProvider brAPIProvider; + private BrAPIClientProvider brAPIClientProvider; + @Property(name = "brapi.server.reference-source") + private String referenceSource; + private Duration requestTimeout; + + private final static String SYSTEM_DEFAULT = BrAPIConstants.SYSTEM_DEFAULT.getValue(); + + @Inject + public ProgramDAOImpl(ProgramDao programDao, DSLContext dsl, BrAPIProvider brAPIProvider, BrAPIClientProvider brAPIClientProvider, + @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout) { + super(programDao); + this.dsl = dsl; + this.brAPIProvider = brAPIProvider; + this.brAPIClientProvider = brAPIClientProvider; + this.requestTimeout = requestTimeout; + } + + @Override + public List get(List programIds){ + return getPrograms(programIds); + } + + @Override + public List get(UUID programId) { + List programList = new ArrayList<>(); + programList.add(programId); + return getPrograms(programList); + } + + @Override + public List getFromEntity(List programEntities){ + List programList = new ArrayList<>(); + for (ProgramEntity programEntity: programEntities){ + programList.add(programEntity.getId()); + } + return getPrograms(programList); + } + + @Override + public List getAll() { + return getPrograms(null); + } + + @Override + public List getActive() { + return getPrograms(null, true); + } + + @Override + public List getProgramByName(String name, boolean caseInsensitive){ + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + Result queryResult; + List resultPrograms = new ArrayList<>(); + + SelectOnConditionStep query = dsl.select() + .from(PROGRAM) + .join(SPECIES).on(PROGRAM.SPECIES_ID.eq(SPECIES.ID)) + .join(createdByUser).on(PROGRAM.CREATED_BY.eq(createdByUser.ID)) + .join(updatedByUser).on(PROGRAM.UPDATED_BY.eq(updatedByUser.ID)); + + if (caseInsensitive){ + queryResult = query + .where(PROGRAM.NAME.equalIgnoreCase(name)) + .fetch(); + } else { + queryResult = query + .where(PROGRAM.NAME.equal(name)) + .fetch(); + } + + return parseProgramQuery(queryResult, createdByUser, updatedByUser); + } + + @Override + public List getProgramByKey(String key){ + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + Result queryResult; + + SelectOnConditionStep query = dsl.select() + .from(PROGRAM) + .join(SPECIES).on(PROGRAM.SPECIES_ID.eq(SPECIES.ID)) + .join(createdByUser).on(PROGRAM.CREATED_BY.eq(createdByUser.ID)) + .join(updatedByUser).on(PROGRAM.UPDATED_BY.eq(updatedByUser.ID)); + + queryResult = query + .where(PROGRAM.KEY.equal(key)) + .fetch(); + + return parseProgramQuery(queryResult, createdByUser, updatedByUser); + } + + private List getPrograms(List programIds) { + return getPrograms(programIds, null); + } + private List getPrograms(List programIds, Boolean active){ + + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + Result queryResult; + List resultPrograms = new ArrayList<>(); + + SelectConditionStep query = dsl.select() + .from(PROGRAM) + .join(SPECIES).on(PROGRAM.SPECIES_ID.eq(SPECIES.ID)) + .join(createdByUser).on(PROGRAM.CREATED_BY.eq(createdByUser.ID)) + .join(updatedByUser).on(PROGRAM.UPDATED_BY.eq(updatedByUser.ID)) + .where("1=1"); + + if (programIds != null){ + query = query + .and(PROGRAM.ID.in(programIds)); + } + + if(active != null) { + query = query.and(PROGRAM.ACTIVE.eq(active)); + } + + queryResult = query.fetch(); + + return parseProgramQuery(queryResult, createdByUser, updatedByUser); + } + + private List parseProgramQuery(Result queryResult, BiUserTable createdByUser, BiUserTable updatedByUser) { + List resultPrograms = new ArrayList<>(); + for (Record record: queryResult){ + if (record.getValue(PROGRAM.BRAPI_URL) == null) { + record.setValue(PROGRAM.BRAPI_URL, SYSTEM_DEFAULT); + } + Program program = Program.parseSQLRecord(record); + // This will do some extra queries, performance may be better in combined query but was having some issues + // getting it working with jooq so went with this for now + program.setNumUsers(getNumProgramUsers(record.getValue(PROGRAM.ID))); + program.setSpecies(Species.parseSQLRecord(record)); + program.setCreatedByUser(User.parseSQLRecord(record, createdByUser)); + program.setUpdatedByUser(User.parseSQLRecord(record, updatedByUser)); + resultPrograms.add(program); + } + + return resultPrograms; + } + + @Override + public int getNumProgramUsers(UUID programId) { + return dsl.selectCount().from(PROGRAM_USER_ROLE) + .where(PROGRAM_USER_ROLE.PROGRAM_ID.eq(programId) + .and(PROGRAM_USER_ROLE.ACTIVE.eq(true))) + .fetchOne(0, Integer.class); + } + + @Override + public ProgramBrAPIEndpoints getProgramBrAPIEndpoints(UUID programId) { + ProgramEntity programEntity = fetchOneById(programId); + + String coreUrl = defaultBrAPICoreUrl; + String genoUrl = defaultBrAPIGenoUrl; + String phenoUrl = defaultBrAPIPhenoUrl; + + // only storing one program brapi url for now so set all to that one + if (!StringUtils.isBlank(programEntity.getBrapiUrl())) { + String brapiUrl = programEntity.getBrapiUrl(); + coreUrl = brapiUrl; + genoUrl = brapiUrl; + phenoUrl = brapiUrl; + } + + return ProgramBrAPIEndpoints.builder() + .coreUrl(Optional.of(coreUrl)) + .genoUrl(Optional.of(genoUrl)) + .phenoUrl(Optional.of(phenoUrl)) + .build(); + } + + @Override + public ProgramEntity fetchOneById(UUID programId) { + return super.fetchOne(PROGRAM.ID, programId); + } + + @Override + public List fetchById(UUID... values) { + return fetch(PROGRAM.ID, values); + } + + @Override + public boolean brapiUrlSupported(String brapiUrl) { + boolean supported = true; + brAPIClientProvider.setBrapiClient(brapiUrl); + ServerInfoApi serverInfoAPI = brAPIProvider.getServerInfoAPI(BrAPIClientType.BRAPI); + + // for now just check for 200 response, in future we could check actual required endpoints + try { + ApiResponse response = serverInfoAPI.serverinfoGet(BrAPIWSMIMEDataTypes.APPLICATION_JSON); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e)); + supported = false; + } + return supported; + } + + @Override + public BrAPIProgram getProgramBrAPI(Program program) { + + ProgramQueryParams searchRequest = new ProgramQueryParams() + .externalReferenceID(program.getId().toString()) + .externalReferenceSource(referenceSource); + + ProgramsApi programsApi = brAPIProvider.getProgramsAPI(BrAPIClientType.CORE); + // Get existing brapi program + ApiResponse brApiPrograms; + try { + brApiPrograms = programsApi.programsGet(searchRequest); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new HttpServerException("Could not find program in BrAPI service."); + } + + if (brApiPrograms.getBody().getResult().getData().isEmpty()) { + throw new HttpServerException("Could not find program in BrAPI service."); + } + + return brApiPrograms.getBody().getResult().getData().get(0); + } + + @Override + public void createProgramBrAPI(Program program) { + + BrAPIExternalReference externalReference = new BrAPIExternalReference() + .referenceID(program.getId().toString()) + .referenceSource(referenceSource); + + BrAPIProgram brApiProgram = new BrAPIProgram() + .programName(program.getName() + " (" + program.getKey() + ")") + .abbreviation(program.getAbbreviation()) + .commonCropName(program.getSpecies().getCommonName()) + .externalReferences(List.of(externalReference)) + .objective(program.getObjective()) + .documentationURL(program.getDocumentationUrl()); + + // POST programs to each brapi service + // TODO: If there is a failure after the first brapi service, roll back all before the failure. + try { + List programsAPIS = brAPIProvider.getAllUniqueProgramsAPI(); + for (ProgramsApi programsAPI: programsAPIS){ + programsAPI.programsPost(List.of(brApiProgram)); + } + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + } + + @Override + public void updateProgramBrAPI(Program program) { + + ProgramQueryParams searchRequest = new ProgramQueryParams() + .externalReferenceID(program.getId().toString()) + .externalReferenceSource(referenceSource); + + // Program goes in all of the clients + // TODO: If there is a failure after the first brapi service, roll back all before the failure. + List programsAPIS = brAPIProvider.getAllUniqueProgramsAPI(); + for (ProgramsApi programsAPI: programsAPIS){ + + // Get existing brapi program + ApiResponse brApiPrograms; + try { + brApiPrograms = programsAPI.programsGet(searchRequest); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new HttpServerException("Could not find program in BrAPI service."); + } + + if (brApiPrograms.getBody().getResult().getData().size() == 0){ + throw new HttpServerException("Could not find program in BrAPI service."); + } + + BrAPIProgram brApiProgram = brApiPrograms.getBody().getResult().getData().get(0); + + //TODO: Need to add archived/not archived when available in brapi + brApiProgram.setProgramName(program.getName() + " (" + program.getKey() + ")"); + brApiProgram.setAbbreviation(program.getAbbreviation()); + brApiProgram.setCommonCropName(program.getSpecies().getCommonName()); + brApiProgram.setObjective(program.getObjective()); + brApiProgram.setDocumentationURL(program.getDocumentationUrl()); + + try { + programsAPI.programsProgramDbIdPut(brApiProgram.getProgramDbId(), brApiProgram); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new HttpServerException("Could not find program in BrAPI service."); + } + } + } + + @Override + public BrAPIClient getCoreClient(UUID programId) { + Program program = get(programId).get(0); + String brapiUrl = !program.getBrapiUrl().equals(SYSTEM_DEFAULT) ? program.getBrapiUrl() : defaultBrAPICoreUrl; + BrAPIClient client = new BrAPIClient(brapiUrl); + initializeHttpClient(client); + return client; + } + + @Override + public BrAPIClient getPhenoClient(UUID programId) { + Program program = get(programId).get(0); + String brapiUrl = !program.getBrapiUrl().equals(SYSTEM_DEFAULT) ? program.getBrapiUrl() : defaultBrAPIPhenoUrl; + BrAPIClient client = new BrAPIClient(brapiUrl); + initializeHttpClient(client); + return client; + } + + private void initializeHttpClient(BrAPIClient brapiClient) { + brapiClient.setHttpClient(brapiClient.getHttpClient() + .newBuilder() + .readTimeout(getRequestTimeout()) + .build()); + } + + //TODO figure out why BrAPIServiceFilterIntegrationTest fails when requestTimeout is set in the constructor + private Duration getRequestTimeout() { + if(requestTimeout != null) { + return requestTimeout; + } + + return Duration.of(5, ChronoUnit.MINUTES); + } +} + diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java new file mode 100644 index 000000000..e86944c25 --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -0,0 +1,684 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.daos.impl; + +import com.github.filosganga.geogson.gson.GeometryAdapterFactory; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.phenotype.VariableQueryParams; +import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.request.BrAPIObservationVariableSearchRequest; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; +import org.breedinginsight.dao.db.tables.BiUserTable; +import org.breedinginsight.dao.db.tables.daos.TraitDao; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.dao.db.tables.records.TraitRecord; +import org.breedinginsight.daos.ObservationDAO; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.TraitDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; +import org.breedinginsight.model.*; +import org.breedinginsight.model.User; +import org.breedinginsight.services.brapi.BrAPIProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; +import org.jooq.*; +import org.jooq.tools.StringUtils; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.breedinginsight.dao.db.Tables.*; +import static org.breedinginsight.services.brapi.BrAPIClientType.PHENO; +import static org.jooq.impl.DSL.lower; + +@Singleton +@Slf4j +public class TraitDAOImpl extends AbstractDAO implements TraitDAO { + + private final DSLContext dsl; + private final BrAPIProvider brAPIProvider; + @Property(name = "brapi.server.reference-source") + private String referenceSource; + @Property(name = "micronaut.bi.api.run-scheduled-tasks") + private Boolean runScheduledTasks; + private final ObservationDAO observationDao; + private final BrAPIDAOUtil brAPIDAOUtil; + private final ProgramCache cache; + private final ProgramDAO programDAO; + private final Gson gson; + + private final static String TAGS_KEY = "tags"; + private final static String FULLNAME_KEY = "fullname"; + + @Inject + public TraitDAOImpl(TraitDao traitDao, DSLContext dsl, BrAPIProvider brAPIProvider, ObservationDAO observationDao, BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO, ProgramCacheProvider programCacheProvider) { + super(traitDao); + this.dsl = dsl; + this.brAPIProvider = brAPIProvider; + this.observationDao = observationDao; + this.brAPIDAOUtil = brAPIDAOUtil; + this.cache = programCacheProvider.getProgramCache(this::populateCache, Trait.class); + this.programDAO = programDAO; + this.gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapterFactory(new GeometryAdapterFactory()) + .create(); + } + + @Scheduled(initialDelay = "2s") + public void setup() { + if(!runScheduledTasks) { + return; + } + // Populate trait cache for all programs on startup + log.debug("Populate traits cache"); + List programs = programDAO.getActive(); + if(programs != null) { + cache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + private Map populateCache(UUID programId) { + List programTraits = fetchTraitsFullByProgramId(programId); + Map traits = new HashMap<>(); + if (!programTraits.isEmpty()) { + programTraits.forEach(trait -> traits.put(trait.getId().toString(), trait)); + } + + return traits; + } + + @Override + public List getTraitsFullByProgramId(UUID programId) { + List programIds = new ArrayList<>(); + programIds.add(programId); + return getTraitsFullByProgramIds(programIds); + } + + @Override + public List getTraitsFullByProgramIds(List programIds) { + List saturatedTraits = new ArrayList<>(); + for(UUID id : programIds) { + Map programTraits = null; + try { + programTraits = cache.get(id); + } catch (ApiException e) { + throw new RuntimeException(e); + } + if(programTraits != null) { + saturatedTraits.addAll(programTraits.values()); + } + } + + return saturatedTraits; + } + + private List fetchTraitsFullByProgramId(UUID programId) { + List saturatedTraits = new ArrayList<>(); + // Get our db traits (equivalent to brapi variables) + List programTraits = getTraitsByProgramIds(programId); + if (programTraits.size() == 0) { + return new ArrayList<>(); + } + + // Get brapi variables + VariableQueryParams variablesRequest = new VariableQueryParams(); + variablesRequest.externalReferenceSource(referenceSource); + variablesRequest.pageSize(10000); + + ApiResponse brApiVariables; + try { + brApiVariables = new ObservationVariablesApi(programDAO.getCoreClient(programId)) + .variablesGet(variablesRequest); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + Map brApiVariableMap = new HashMap<>(); + for (BrAPIObservationVariable brApiVariable : brApiVariables.getBody() + .getResult() + .getData()) { + List brApiExternalReferences = brApiVariable.getExternalReferences(); + for (BrAPIExternalReference brApiExternalReference : brApiExternalReferences) { + if (brApiExternalReference.getReferenceID() != null) { + brApiVariableMap.put(brApiExternalReference.getReferenceID(), brApiVariable); + } + } + } + + for (Trait trait : programTraits) { + // assumes external reference id is unique to each brapi variable + if (brApiVariableMap.containsKey(trait.getId() + .toString())) { + BrAPIObservationVariable brApiVariable = brApiVariableMap.get(trait.getId() + .toString()); + saturateTrait(trait, brApiVariable); + saturatedTraits.add(trait); + } else { + throw new InternalServerException("Could not find trait in returned brapi server results"); + } + } + + return saturatedTraits; + } + + @Override + public List getTraitsByProgramId(UUID programId) { + return getTraitsByProgramIds(programId); + } + + @Override + public List getTraitsByProgramIds(UUID... programIds) { + + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + + Result recordResult = getTraitSql(createdByUser, updatedByUser) + .where(PROGRAM_ONTOLOGY.PROGRAM_ID.in(programIds)) + .fetch(); + + List traitResults = new ArrayList<>(); + for (Record record: recordResult) { + Trait trait = parseTraitRecord(record, createdByUser, updatedByUser); + traitResults.add(trait); + } + + return traitResults; + } + + @Override + public List getTraitsById(UUID... traitIds){ + + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + + Result recordResult = getTraitSql(createdByUser, updatedByUser) + .and(TRAIT.ID.in(traitIds)) + .fetch(); + + List traitResults = new ArrayList<>(); + for (Record record: recordResult) { + Trait trait = parseTraitRecord(record, createdByUser, updatedByUser); + traitResults.add(trait); + } + + return traitResults; + } + + @Override + public Optional getTraitFull(UUID programId, UUID traitId){ + + try { + return Optional.ofNullable(cache.get(programId).get(traitId.toString())); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + } + + // could be more efficient to do a single get instead of search in saved search case but less code this way + // and search stuff is working in breedbase + @Override + public List getObservationsForTrait(UUID traitId) { + return getObservationsForTraits(Stream.of(traitId).collect(Collectors.toList())); + } + + @Override + public List getObservationsForTraits(List traitIds) { + + List ids = traitIds.stream() + .map(UUID::toString) + .collect(Collectors.toList()); + + List variables = searchVariables(ids); + + // TODO: make sure have all expected external references + if (variables.size() != ids.size()) { + throw new InternalServerException("Observation variables search results mismatch"); + } + + List brapiVariableIds = variables.stream() + .map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); + + return observationDao.getObservationsByVariableDbIds(brapiVariableIds); + } + + @Override + public List getObservationsForTraitsByBrAPIProgram(String brapiProgramId, List traitIds) { + + List ids = traitIds.stream() + .map(UUID::toString) + .collect(Collectors.toList()); + + List variables = searchVariables(ids); + + // TODO: make sure have all expected external references + if (variables.size() != ids.size()) { + throw new InternalServerException("Observation variables search results mismatch"); + } + + List brapiVariableIds = variables.stream() + .map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); + + return observationDao.getObservationsByVariableAndBrAPIProgram(brapiProgramId, brapiVariableIds); + } + + @Override + public List searchVariables(List variableIds) { + + if (variableIds == null || variableIds.size() == 0) return new ArrayList<>(); + try { + BrAPIObservationVariableSearchRequest request = new BrAPIObservationVariableSearchRequest() + .externalReferenceIDs(variableIds); + + ObservationVariablesApi api = brAPIProvider.getVariablesAPI(PHENO); + return brAPIDAOUtil.search( + api::searchVariablesPost, + api::searchVariablesSearchResultsDbIdGet, + request + ); + } catch (ApiException e) { + throw new InternalServerException("Observation variables brapi search error", e); + } + } + + @Override + public Optional getTrait(UUID programId, UUID traitId) { + + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + + Record record = getTraitSql(createdByUser, updatedByUser) + .where(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(programId)) + .and(TRAIT.ID.eq(traitId)) + .fetchOne(); + + if (record == null) { + return Optional.empty(); + } + + return Optional.of(parseTraitRecord(record, createdByUser, updatedByUser)); + } + + @Override + public List createTraitsBrAPI(List traits, User actingUser, Program program){ + + //TODO: Pass ontology reference + + // Convert our traits into BrAPI traits + List brApiVariables = new ArrayList<>(); + for (Trait trait: traits) { + + // Construct method + BrAPIExternalReference methodReference = new BrAPIExternalReference() + .referenceID(trait.getMethod().getId().toString()) + .referenceSource(referenceSource); + BrAPIMethod brApiMethod = new BrAPIMethod() + .methodName(constructMethodName(trait, program)) + .externalReferences(List.of(methodReference)) + .methodClass(trait.getMethod().getMethodClass()) + .description(trait.getMethod().getDescription()) + .formula(trait.getMethod().getFormula()); + + // Construct scale + BrAPIExternalReference scaleReference = new BrAPIExternalReference() + .referenceID(trait.getScale().getId().toString()) + .referenceSource(referenceSource); + BrAPITraitDataType brApiTraitDataType = BrAPITraitDataType.valueOf(trait.getScale().getDataType().toString()); + BrAPIScaleValidValues brApiScaleValidValues = new BrAPIScaleValidValues() + .categories(trait.getScale().getCategories()) + .max(trait.getScale().getValidValueMax()) + .min(trait.getScale().getValidValueMin()); + BrAPIScale brApiScale = new BrAPIScale() + .scaleName(String.format("%s [%s]", trait.getScale().getScaleName(), program.getKey())) + .externalReferences(List.of(scaleReference)) + .dataType(brApiTraitDataType) + .decimalPlaces(trait.getScale().getDecimalPlaces()) + .validValues(brApiScaleValidValues); + + // Construct trait + BrAPIExternalReference traitReference = new BrAPIExternalReference() + .referenceID(trait.getId().toString()) + .referenceSource(referenceSource); + BrAPITrait brApiTrait = new BrAPITrait() + .traitName(String.format("%s %s [%s]", trait.getEntity(), trait.getAttribute(), program.getKey())) + .traitDescription(trait.getTraitDescription()) + .synonyms(trait.getSynonyms()) + .status("active") + .entity(trait.getEntity()) + .mainAbbreviation(trait.getMainAbbreviation()) + .traitClass(trait.getTraitClass()) + .externalReferences(List.of(traitReference)) + .attribute(trait.getAttribute()); + + BrAPIExternalReference variableReference = new BrAPIExternalReference() + .referenceID(trait.getId().toString()) + .referenceSource(referenceSource); + BrAPIExternalReference programReference = new BrAPIExternalReference() + .referenceID(program.getId().toString()) + .referenceSource(referenceSource+"/programs"); + BrAPIObservationVariable brApiVariable = new BrAPIObservationVariable() + .method(brApiMethod) + .scale(brApiScale) + .trait(brApiTrait) + .externalReferences(List.of(variableReference, programReference)) + .observationVariableName(String.format("%s [%s]", trait.getObservationVariableName(), program.getKey())) + .status("active") + .language("english") + .scientist(actingUser.getName()) + .defaultValue(trait.getDefaultValue()) + .synonyms(trait.getSynonyms()) + .institution(program.getName()) + .commonCropName(program.getSpecies().getCommonName()); + if (trait.getTags() != null) brApiVariable.putAdditionalInfoItem(TAGS_KEY, trait.getTags()); + if (trait.getFullName() != null) brApiVariable.putAdditionalInfoItem(FULLNAME_KEY, trait.getFullName()); + + if (trait.getActive() == null || trait.getActive()){ + brApiVariable.setStatus("active"); + } else { + brApiVariable.setStatus("archived"); + } + + + // Unused + //.contextOfUse() + //.documentationURL() + //.growthStage() + + brApiVariables.add(brApiVariable); + } + + + // POST variables to each brapi service + // TODO: If there is a failure after the first brapi service, roll back all before the failure. + ApiResponse createdVariables = null; + try { + List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); + for (ObservationVariablesApi variablesAPI: variablesAPIS){ + createdVariables = variablesAPI.variablesPost(brApiVariables); + } + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Error making BrAPI call", e); + } + + if(createdVariables == null) { + throw new InternalServerException("Creating new variable did not return any data"); + } + + // Pull our traits from the db + List traitIds = traits.stream().map(TraitEntity::getId).collect(Collectors.toList()); + List createdTraits = getTraitsById(traitIds.toArray(UUID[]::new)); + + // Saturate our traits from the brapi return information + for (Trait trait: createdTraits){ + for (BrAPIObservationVariable variable: createdVariables.getBody().getResult().getData()){ + if (variable.getExternalReferences() != null) { + for (BrAPIExternalReference brApiExternalReference: variable.getExternalReferences()){ + if (brApiExternalReference.getReferenceSource().equals(referenceSource) && + brApiExternalReference.getReferenceID().equals(trait.getId().toString())){ + saturateTrait(trait, variable); + + cache.set(program.getId(), trait.getId().toString(), trait); + } + } + } + } + } + + return createdTraits; + } + + @Override + public Trait updateTraitBrAPI(Trait trait, Program program) { + //TODO: Need to roll back somehow if there is an error + Trait updatedTrait = null; + List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); + for (ObservationVariablesApi variablesAPI: variablesAPIS){ + // GET brapi trait + BrAPIObservationVariable existingVariable = getBrAPIVariable(variablesAPI, trait.getId()); + + // Change method + existingVariable.getMethod().setMethodName(constructMethodName(trait, program)); + existingVariable.getMethod().setMethodClass(trait.getMethod().getMethodClass()); + existingVariable.getMethod().setDescription(trait.getMethod().getDescription()); + existingVariable.getMethod().setFormula(trait.getMethod().getFormula()); + + // Change scale + BrAPITraitDataType brApiTraitDataType = BrAPITraitDataType.valueOf(trait.getScale().getDataType().toString()); + existingVariable.getScale().setScaleName(String.format("%s [%s]", trait.getScale().getScaleName(), program.getKey())); + existingVariable.getScale().setDataType(brApiTraitDataType); + existingVariable.getScale().setDecimalPlaces(trait.getScale().getDecimalPlaces()); + BrAPIScaleValidValues brApiScaleValidValues = new BrAPIScaleValidValues() + .categories(trait.getScale().getCategories()) + .max(trait.getScale().getValidValueMax()) + .min(trait.getScale().getValidValueMin()); + existingVariable.getScale().setValidValues(brApiScaleValidValues); + + // Change trait + existingVariable.getTrait().setTraitName(String.format("%s %s [%s]", trait.getEntity(), trait.getAttribute(), program.getKey())); + existingVariable.getTrait().setTraitDescription(trait.getTraitDescription()); + existingVariable.getTrait().setSynonyms(trait.getSynonyms()); + existingVariable.getTrait().setEntity(trait.getProgramObservationLevel().getName()); + existingVariable.getTrait().setMainAbbreviation(trait.getMainAbbreviation()); + existingVariable.getTrait().setTraitClass(trait.getTraitClass()); + existingVariable.getTrait().setAttribute(trait.getAttribute()); + + // Change variable + existingVariable.setObservationVariableName(String.format("%s [%s]", trait.getObservationVariableName(), program.getKey())); + existingVariable.setDefaultValue(trait.getDefaultValue()); + existingVariable.setSynonyms(trait.getSynonyms()); + if (trait.getActive() == null || trait.getActive()){ + existingVariable.setStatus("active"); + } else { + existingVariable.setStatus("archived"); + } + existingVariable.putAdditionalInfoItem(TAGS_KEY, trait.getTags()); + if (trait.getFullName() != null) existingVariable.putAdditionalInfoItem(FULLNAME_KEY, trait.getFullName()); + + // PUT brapi trait + BrAPIObservationVariable updatedVariable = putBrAPIVariable(variablesAPI, existingVariable); + + // Retrieve our update trait from the db + updatedTrait = getTrait(program.getId(), trait.getId()).get(); + saturateTrait(updatedTrait, updatedVariable); + + //update cache + cache.set(program.getId(), updatedTrait.getId().toString(), updatedTrait); + } + + return updatedTrait; + } + + private String constructMethodName(Trait trait, Program program) { + return !StringUtils.isBlank(trait.getMethod().getDescription()) ? + String.format("%s %s [%s]", trait.getMethod().getDescription(), trait.getMethod().getMethodClass(), program.getKey()) : + String.format("%s [%s]", trait.getMethod().getMethodClass(), program.getKey()); + } + + private BrAPIObservationVariable getBrAPIVariable(ObservationVariablesApi variablesAPI, UUID traitId) { + + BrAPIObservationVariable existingVariable; + try { + VariableQueryParams queryParams = new VariableQueryParams(); + queryParams.externalReferenceID(traitId.toString()); + queryParams.externalReferenceSource(referenceSource); + ApiResponse existingVariableResponse = + variablesAPI.variablesGet(queryParams); + List variableList = existingVariableResponse.getBody().getResult().getData(); + if (variableList.size() == 1) { + existingVariable = variableList.get(0); + } else { + throw new InternalServerException(String.format("Unable to find variable with id %s in brapi server.", traitId.toString())); + } + + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException(String.format("Unable to retrieve variable with id %s in brapi server.", traitId.toString())); + } + + return existingVariable; + } + + private BrAPIObservationVariable putBrAPIVariable(ObservationVariablesApi variablesAPI, BrAPIObservationVariable variable) { + + BrAPIObservationVariable updatedVariable; + try { + ApiResponse updatedResponse = + variablesAPI.variablesObservationVariableDbIdPut(variable.getObservationVariableDbId(), variable); + updatedVariable = updatedResponse.getBody().getResult(); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw new InternalServerException("Unable to save variable in brapi server."); + } + return updatedVariable; + } + + private Trait parseTraitRecord(Record record, BiUserTable createdByUser, BiUserTable updatedByUser) { + Trait trait = Trait.parseSqlRecord(record); + Scale scale = Scale.parseSqlRecord(record); + Method method = Method.parseSqlRecord(record); + ProgramOntology programOntology = ProgramOntology.parseSqlRecord(record); + ProgramObservationLevel programObservationLevel = ProgramObservationLevel.parseSqlRecord(record); + User createUser = User.parseSQLRecord(record, createdByUser); + User updateUser = User.parseSQLRecord(record, updatedByUser); + + trait.setScale(scale); + trait.setMethod(method); + trait.setProgramOntology(programOntology); + trait.setProgramObservationLevel(programObservationLevel); + trait.setCreatedByUser(createUser); + trait.setUpdatedByUser(updateUser); + + return trait; + } + + private SelectOnConditionStep getTraitSql(BiUserTable createdByTableAlias, BiUserTable updatedByTableAlias) { + return dsl.select() + .from(TRAIT) + .join(METHOD).on(TRAIT.METHOD_ID.eq(METHOD.ID)) + .join(SCALE).on(TRAIT.SCALE_ID.eq(SCALE.ID)) + .join(PROGRAM_OBSERVATION_LEVEL).on(TRAIT.PROGRAM_OBSERVATION_LEVEL_ID.eq(PROGRAM_OBSERVATION_LEVEL.ID)) + .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) + .join(createdByTableAlias).on(TRAIT.CREATED_BY.eq(createdByTableAlias.ID)) + .join(updatedByTableAlias).on(TRAIT.UPDATED_BY.eq(updatedByTableAlias.ID)); + } + + @Override + public List getTraitsByTraitName(UUID programId, List traits){ + + String[] names = traits.stream() + .filter(trait -> trait.getObservationVariableName() != null) + .map(trait -> trait.getObservationVariableName().toLowerCase()) + .collect(Collectors.toList()).toArray(String[]::new); + + List traitResults = new ArrayList<>(); + if (names.length > 0){ + + Result records = dsl.select() + .from(TRAIT) + .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) + .join(PROGRAM).on(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(PROGRAM.ID)) + .join(SCALE).on(TRAIT.SCALE_ID.eq(SCALE.ID)) + .join(METHOD).on(TRAIT.METHOD_ID.eq(METHOD.ID)) + .where(PROGRAM.ID.eq(programId)) + .and(lower(TRAIT.OBSERVATION_VARIABLE_NAME).in(names)) + .fetch(); + + for (Record record: records) { + Trait trait = Trait.parseSqlRecord(record); + Scale scale = Scale.parseSqlRecord(record); + Method method = Method.parseSqlRecord(record); + trait.setScale(scale); + trait.setMethod(method); + traitResults.add(trait); + } + } + + return traitResults; + } + + @Override + public List getTraitsByAbbreviation(UUID programId, List abbreviations) { + + Result records = dsl.select() + .from(TRAIT) + .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) + .join(PROGRAM).on(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(PROGRAM.ID)) + .and(PROGRAM.ID.eq(programId)) + .fetch(); + + List traitResults = new ArrayList<>(); + for (Record record: records) { + Trait trait = Trait.parseSqlRecord(record); + traitResults.add(trait); + } + + return traitResults; + } + + @Override + public TraitEntity fetchOneById(UUID id) { + return super.fetchOne(TRAIT.ID, id); + } + + @Override + public List fetchById(UUID traitId) { + return super.fetch(TRAIT.ID, traitId); + } + + private void saturateTrait(Trait trait, BrAPIObservationVariable brApiVariable) { + + if (brApiVariable.getAdditionalInfo() != null) { + List tags = null; + String fullName = null; + if (brApiVariable.getAdditionalInfo().has(TAGS_KEY) && !brApiVariable.getAdditionalInfo().get(TAGS_KEY).isJsonNull()) { + tags = gson.fromJson(brApiVariable.getAdditionalInfo().getAsJsonArray(TAGS_KEY), List.class); + } + if (brApiVariable.getAdditionalInfo().has(FULLNAME_KEY) && !brApiVariable.getAdditionalInfo().get(FULLNAME_KEY).isJsonNull()) { + fullName = brApiVariable.getAdditionalInfo().get(FULLNAME_KEY).getAsString(); + } + trait.setBrAPIProperties(brApiVariable, tags, fullName); + } else { + trait.setBrAPIProperties(brApiVariable); + } + + Method method = trait.getMethod(); + method.setBrAPIProperties(brApiVariable.getMethod()); + + Scale scale = trait.getScale(); + scale.setBrAPIProperties(brApiVariable.getScale()); + } +} diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 70f218049..d10a22b7c 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -74,17 +74,6 @@ public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, Pro this.brAPIClientProvider = brAPIClientProvider; } - @Inject - public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, - ProgramObservationLevelDAO programObservationLevelDAO, SpeciesService speciesService, - DSLContext dsl) { - this.dao = dao; - this.programOntologyDAO = programOntologyDAO; - this.programObservationLevelDAO = programObservationLevelDAO; - this.speciesService = speciesService; - this.dsl = dsl; - } - public Optional getById(UUID programId) { List programs = dao.get(programId); diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index dc737c131..e1068c354 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -22,6 +22,7 @@ import io.reactivex.functions.Consumer; import io.reactivex.functions.Function; import io.reactivex.functions.Function3; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; @@ -35,6 +36,7 @@ import java.util.Optional; @Singleton +@Slf4j public class BrAPIDAOUtil { @Property(name = "brapi.search.wait-time") @@ -115,6 +117,7 @@ public List search(Funct return listResult; } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); throw e; } catch (Exception e) { throw new InternalServerException(e.toString(), e); @@ -176,6 +179,7 @@ public List post(List brapiObjects, return listResult; } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); throw e; } catch (Exception e) { throw new InternalServerException(e.toString(), e); diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index f79af13fb..bebfd7031 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -18,6 +18,7 @@ package org.breedinginsight.utilities; import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import java.util.List; @@ -102,4 +103,16 @@ public static String removeProgramKeyAndUnknownAdditionalData(String original, S String stripped = original.replaceAll(keyValueRegEx, ""); return stripped; } + + public static String generateApiExceptionLogMessage(ApiException e) { + return new StringBuilder("BrAPI Exception: \n\t").append("message: ") + .append(e.getMessage()) + .append("\n\t") + .append("body: ") + .append(e.getResponseBody()) + .append("\n\t") + .append("code: ") + .append(e.getCode()) + .toString(); + } } diff --git a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java index f56fe15f2..dc77c0479 100644 --- a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java +++ b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java @@ -169,7 +169,7 @@ private static List sort(List data, QueryParams queryParams, AbstractQueryMapper return data; } - private static List search(List data, SearchRequest searchRequest, AbstractQueryMapper mapper) { + private static List search(List data, SearchRequest searchRequest, AbstractQueryMapper mapper) { List filterFields = new ArrayList<>(); if (searchRequest.getFilters() != null) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2c4f8179e..fa370b90d 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -34,6 +34,7 @@ micronaut: bi: api: version: v1 + run-scheduled-tasks: true security: enabled: true oauth2: @@ -170,4 +171,10 @@ email: port: ${EMAIL_RELAY_PORT} from: ${EMAIL_FROM} - +redisson: + single-server-config: + address: ${REDIS_URL:`redis://localhost:6379`} + connect-timeout: ${REDIS_TIMEOUT:30s} + ssl-enable-endpoint-identification: ${REDIS_SSL:false} + threads: 16 + netty-threads: 32 \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index fb2ca5583..94f2bb4fd 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -19,9 +19,11 @@ import lombok.Getter; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.jooq.impl.DSL; +import org.junit.jupiter.api.AfterAll; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -31,36 +33,40 @@ import java.sql.Connection; import java.sql.DriverManager; import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.HashMap; import java.util.Map; +@Slf4j public class BrAPITest extends DatabaseTest { @Getter - private static GenericContainer brapiContainer; + private GenericContainer brapiContainer; + + private Connection con; @Getter private DSLContext brapiDsl; @SneakyThrows public BrAPITest() { - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server") - .withNetwork(super.getDbContainer().getNetwork()) + super(); + + brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") + .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.alwaysPull()) .withExposedPorts(8080) .withEnv("BRAPI_DB_SERVER", - String.format("%s:%s", - super.getDbContainer().getNetworkAliases().get(0), - 5432)) + String.format("%s:%s", + "testdb", + 5432)) .withEnv("BRAPI_DB", "postgres") .withEnv("BRAPI_DB_USER", "postgres") .withEnv("BRAPI_DB_PASSWORD", "postgres") .withClasspathResourceMapping("brapi/properties/application.properties", "/home/brapi/properties/application.properties", BindMode.READ_ONLY) - .waitingFor(Wait.forHttp("/brapi/v2/serverinfo").forStatusCode(200).withHeader("Accept", "application/json").withStartupTimeout(Duration.of(3, ChronoUnit.MINUTES))); + .waitingFor(Wait.forLogMessage(".*: Started BrapiTestServer in \\d*.\\d* seconds.*", 1).withStartupTimeout(Duration.ofMinutes(1))); + brapiContainer.start(); // Get a dsl connection for the brapi db - Connection con = DriverManager. + con = DriverManager. getConnection(String.format("jdbc:postgresql://%s:%s/postgres", super.getDbContainer().getContainerIpAddress(), super.getDbContainer().getMappedPort(5432)), "postgres", "postgres"); @@ -70,7 +76,7 @@ public BrAPITest() { @Nonnull @Override public Map getProperties() { - Map properties = new HashMap<>(); + Map properties = super.getProperties(); Integer containerPort = brapiContainer.getMappedPort(8080); String containerIp = brapiContainer.getContainerIpAddress(); @@ -78,14 +84,15 @@ public Map getProperties() { properties.put("brapi.server.core-url", String.format("http://%s:%s/", containerIp, containerPort)); properties.put("brapi.server.geno-url", String.format("http://%s:%s/", containerIp, containerPort)); properties.put("brapi.server.pheno-url", String.format("http://%s:%s/", containerIp, containerPort)); - properties.putAll(super.getProperties()); return properties; } - public void stopContainers() { + @SneakyThrows + @AfterAll + public void stopBrApiContainer() { + con.close(); brapiContainer.stop(); - super.stopContainers(); } } diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index 107fffe93..a2ed3650b 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -20,33 +20,65 @@ import io.micronaut.test.support.TestPropertyProvider; import lombok.Getter; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.PullPolicy; import javax.annotation.Nonnull; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; +@Slf4j public class DatabaseTest implements TestPropertyProvider { + private static final String dbName = "bitest"; + private static final String dbPassword = "postgres"; + + @Getter + private GenericContainer dbContainer; + + @Getter + private Network network; + @Getter - private static GenericContainer dbContainer; - private final String dbName = "bitest"; - private final String dbPassword = "postgres"; + private GenericContainer redisContainer; + + @Getter + private RedissonClient redisConnection; @SneakyThrows public DatabaseTest() { + network = Network.newNetwork(); dbContainer = new GenericContainer<>("postgres:11.4") - .withNetwork(Network.newNetwork()) - .withNetworkAliases("test-db") + .withNetwork(network) + .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) .withExposedPorts(5432) .withEnv("POSTGRES_DB", dbName) .withEnv("POSTGRES_PASSWORD", dbPassword) - .waitingFor(Wait.forListeningPort()); + .waitingFor(Wait.forLogMessage(".*LOG: database system is ready to accept connections.*", 1).withStartupTimeout(Duration.of(2, ChronoUnit.MINUTES))); dbContainer.start(); + redisContainer = new GenericContainer<>("redis") + .withNetwork(network) + .withNetworkAliases("redis") + .withImagePullPolicy(PullPolicy.defaultPolicy()) + .withExposedPorts(6379) + .waitingFor(Wait.forListeningPort()); + redisContainer.start(); + + Integer redisContainerPort = redisContainer.getMappedPort(6379); + String redisContainerIp = redisContainer.getContainerIpAddress(); + Config redissonConfig = new Config(); + redissonConfig.useSingleServer().setAddress(String.format("redis://%s:%s", redisContainerIp, redisContainerPort)); + redisConnection = Redisson.create(redissonConfig); } @Nonnull @@ -55,16 +87,24 @@ public Map getProperties() { Map properties = new HashMap<>(); Integer containerPort = dbContainer.getMappedPort(5432); String containerIp = dbContainer.getContainerIpAddress(); - properties.put("datasources.default.url", String.format("jdbc:postgresql://%s:%s/%s", containerIp, containerPort, dbName)); - return properties; - } + properties.put("micronaut.bi.api.run-scheduled-tasks", "false"); + properties.put("datasources.default.initialization-fail-timeout", "10"); - public GenericContainer getDbContainer() { - return dbContainer; + Integer redisContainerPort = redisContainer.getMappedPort(6379); + String redisContainerIp = redisContainer.getContainerIpAddress(); + properties.put("redisson.single-server-config.address", String.format("redis://%s:%s", redisContainerIp, redisContainerPort)); + + properties.put("micronaut.http.client.read-timeout", "1m"); + + return properties; } - protected void stopContainers() { + @SneakyThrows + @AfterAll + public void stopContainers() { + redisContainer.stop(); dbContainer.stop(); + network.close(); } } diff --git a/src/test/java/org/breedinginsight/api/auth/AuthServiceLoginHandlerUnitTest.java b/src/test/java/org/breedinginsight/api/auth/AuthServiceLoginHandlerUnitTest.java index e7b7f1eef..93b324484 100644 --- a/src/test/java/org/breedinginsight/api/auth/AuthServiceLoginHandlerUnitTest.java +++ b/src/test/java/org/breedinginsight/api/auth/AuthServiceLoginHandlerUnitTest.java @@ -51,11 +51,6 @@ public class AuthServiceLoginHandlerUnitTest extends DatabaseTest { private String userName = "1111-2222-3333-4444"; - @AfterAll - public void finish() { - super.stopContainers(); - } - @Test public void returnsDefaultBadUrl() { diff --git a/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java b/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java index 5daa38a41..26a53b9ca 100644 --- a/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/auth/rules/ProgramSecuredAnnotationRuleIntegrationTest.java @@ -82,11 +82,6 @@ public class ProgramSecuredAnnotationRuleIntegrationTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); - @AfterAll - public void finish() { - super.stopContainers(); - } - @BeforeAll void setup() { fp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); diff --git a/src/test/java/org/breedinginsight/api/model/v1/request/UserRequestTest.java b/src/test/java/org/breedinginsight/api/model/v1/request/UserRequestTest.java index a2796c869..9a0846e19 100644 --- a/src/test/java/org/breedinginsight/api/model/v1/request/UserRequestTest.java +++ b/src/test/java/org/breedinginsight/api/model/v1/request/UserRequestTest.java @@ -36,11 +36,6 @@ public class UserRequestTest extends DatabaseTest { @Inject private Validator validator; - @AfterAll - public void finish() { - super.stopContainers(); - } - @Test void validRequest() { UserRequest request = new UserRequest(); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/AccessibilityControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/AccessibilityControllerIntegrationTest.java index 4d31f4561..b111c6136 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/AccessibilityControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/AccessibilityControllerIntegrationTest.java @@ -51,9 +51,6 @@ public class AccessibilityControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test @Order(1) // Expects at least one valid accessibility in the database to pass diff --git a/src/test/java/org/breedinginsight/api/v1/controller/CountryControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/CountryControllerIntegrationTest.java index c9555c06d..801361125 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/CountryControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/CountryControllerIntegrationTest.java @@ -53,9 +53,6 @@ public class CountryControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test @Order(1) // Expects at least one valid country in the database to pass diff --git a/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java index fcc5b4039..a8100835a 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java @@ -51,9 +51,6 @@ public class EnvironmentDataTypeControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test @Order(1) // Expects at least one valid environmentType in the database to pass diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java index 47115ce10..378e99085 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ImportControllerIntegrationTest.java @@ -79,9 +79,6 @@ public class ImportControllerIntegrationTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll @SneakyThrows public void setup() { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java b/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java index 5aed4b3b5..4fc66e151 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/InternalServerErrorHandlerUnitTest.java @@ -107,9 +107,6 @@ EnvironmentTypeService environmentTypeService() { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll void setup() { var securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java index fdf7ab39f..6949499e9 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/JobControllerIntegrationTest.java @@ -114,9 +114,6 @@ public void setup() { } } - @AfterAll - public void finish() { super.stopContainers(); } - @Test public void fetchJobs() throws DoesNotExistException { try { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java index 82195581c..bf8dde632 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/OntologyControllerIntegrationTest.java @@ -65,9 +65,6 @@ public class OntologyControllerIntegrationTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll void setup() throws Exception { // Create two programs with fanny pack diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java index e67b44fd9..44727dbaf 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ProgramControllerIntegrationTest.java @@ -154,9 +154,6 @@ public class ProgramControllerIntegrationTest extends BrAPITest { @MockBean(bean = EmailUtil.class) EmailUtil emailUtil() { return mock(EmailUtil.class); } - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll void setup() throws Exception { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/RoleControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/RoleControllerIntegrationTest.java index 70307fa7e..f5305b6e8 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/RoleControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/RoleControllerIntegrationTest.java @@ -53,9 +53,6 @@ public class RoleControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test @Order(1) // Expects at least one valid role in the database to pass diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ServerInfoControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ServerInfoControllerIntegrationTest.java index 9807e14ea..d41ebe1db 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ServerInfoControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ServerInfoControllerIntegrationTest.java @@ -44,9 +44,6 @@ public class ServerInfoControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test public void getVersionInfo() throws IOException { Flowable> call = client.exchange(GET("/server-info"), String.class); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TokenControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TokenControllerIntegrationTest.java index 43fd5a6d3..7be64187c 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TokenControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TokenControllerIntegrationTest.java @@ -41,9 +41,6 @@ public class TokenControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test void getApiTokenMissingRequiredParameter() { Flowable> call = client.exchange( diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TopographyControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TopographyControllerIntegrationTest.java index 9255d7373..dd2881244 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TopographyControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TopographyControllerIntegrationTest.java @@ -51,9 +51,6 @@ public class TopographyControllerIntegrationTest extends DatabaseTest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } - @Test @Order(1) // Expects at least one valid topography in the database to pass diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java index bb576dc21..b851af41e 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java @@ -26,6 +26,8 @@ import io.micronaut.http.client.annotation.Client; import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.test.annotation.MockBean; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import junit.framework.AssertionFailedError; @@ -39,7 +41,6 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationVariable; import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; -import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponseResult; @@ -56,20 +57,25 @@ import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.daos.cache.FetchFunction; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.*; import org.breedinginsight.services.SpeciesService; -import org.breedinginsight.services.TraitService; -import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; +import org.mockito.stubbing.Answer; +import org.redisson.api.RedissonClient; import javax.inject.Inject; import java.time.OffsetDateTime; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import static io.micronaut.http.HttpRequest.*; import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; @MicronautTest @@ -88,12 +94,21 @@ public class TraitControllerIntegrationTest extends BrAPITest { @Inject private TraitDao traitDao; @Inject - private TraitService traitService; - @Inject private UserDAO userDAO; @Inject private SpeciesService speciesService; + @Inject + private RedissonClient redissonClient; + + @Inject + private ProgramCacheProvider programCacheProvider; + + @MockBean(ProgramCacheProvider.class) + ProgramCacheProvider programCacheProvider() { + return mock(ProgramCacheProvider.class); + } + private Species validSpecies; private List validTraits; private Program validProgram; @@ -109,8 +124,28 @@ public class TraitControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; - @AfterAll - public void finish() { super.stopContainers(); } + private AtomicReference> fetchException = new AtomicReference<>(Optional.empty()); + @BeforeEach + public void beforeEach() { + fetchException = new AtomicReference<>(Optional.empty()); + when(programCacheProvider.getProgramCache(isA(FetchFunction.class), any(Class.class))).thenAnswer((Answer) invocation -> { + FetchFunction fetchFunction = invocation.getArgument(0); + return new ProgramCache(redissonClient, uuid -> { + try { + return fetchFunction.apply(uuid); + } catch (Exception e) { + fetchException.set(Optional.of(e)); + throw e; + } + }, invocation.getArgument(1)); + }); + } + + @AfterEach + public void afterEach() { + redissonClient.getKeys().deleteByPattern("*:trait"); + reset(programCacheProvider); + } @BeforeAll @SneakyThrows @@ -200,15 +235,13 @@ public Species getTestSpecies() { @SneakyThrows @Order(1) public void getTraitsNoExistInBrAPI() { - - Flowable> call = client.exchange( + client.exchange( GET("/programs/" + validProgram.getId() + "/traits?full=true").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + ).blockingFirst(); - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus()); + assertTrue(fetchException.get().isPresent(), "No exception was thrown"); + assertTrue(fetchException.get().get() instanceof InternalServerException, "Wrong exception type was thrown"); + assertEquals("Could not find trait in returned brapi server results", fetchException.get().get().getMessage()); } @Test @@ -223,7 +256,7 @@ public void getTraitTagsDefaultFavorites() { HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.OK, response.getStatus()); - JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + JsonObject result = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject().getAsJsonObject("result"); JsonArray data = result.getAsJsonArray("data"); assertEquals(1, data.size(), "Wrong number of results returned."); @@ -242,7 +275,11 @@ public void getTraitSingleNoExistInBrAPI() { HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { HttpResponse response = call.blockingFirst(); }); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus()); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + + assertTrue(fetchException.get().isPresent(), "No exception was thrown"); + assertTrue(fetchException.get().get() instanceof InternalServerException, "Wrong exception type was thrown"); + assertEquals("Could not find trait in returned brapi server results", fetchException.get().get().getMessage()); } //region POST traits @@ -563,7 +600,7 @@ public void getTraitTags() { JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); JsonArray data = result.getAsJsonArray("data"); - assertEquals(3, data.size(), "Wrong number of results returned."); + assertEquals(3, data.size(), "Wrong number of results returned: " + data.toString()); Set tagsList = new HashSet<>(List.of("favorites", "leaf trait", "stem trait")); Set foundTags = new HashSet<>(); for (JsonElement tagElement: data) { @@ -1204,6 +1241,7 @@ public void postTraitOrdinalMissingCategoryVariables() { assertEquals("scale.categories.label", labelError.get("field").getAsString(), "wrong error returned"); } + @SneakyThrows @Test @Order(11) public void getTraitsQuery() { @@ -1235,6 +1273,9 @@ public void getTraitsQuery() { HttpResponse createResponse = createCall.blockingFirst(); assertEquals(HttpStatus.OK, createResponse.getStatus()); + //let the cache repopulate + Thread.sleep(3000); + List allTraits = traitDao.findAll(); Flowable> call = client.exchange( diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java index 77bfa672d..1052a969c 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java @@ -95,9 +95,6 @@ public class TraitUploadControllerIntegrationTest extends BrAPITest { String invalidUUID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; String invalidProgram = invalidUUID; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll public void setup() { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java index a2242e47d..6549586e0 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UploadControllerIntegrationTest.java @@ -77,9 +77,6 @@ public class UploadControllerIntegrationTest extends BrAPITest { (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll @SneakyThrows public void setup() { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java index 6f2705488..23fcfa32c 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/UserControllerIntegrationTest.java @@ -100,9 +100,6 @@ public class UserControllerIntegrationTest extends DatabaseTest { private SystemRoleEntity validSystemRole; private String invalidUUID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll void setup() { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java index 0af3f0172..35d120933 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/brapi/BrAPIServiceFilterIntegrationTest.java @@ -26,10 +26,10 @@ import io.micronaut.http.client.annotation.Client; import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.http.netty.cookies.NettyCookie; -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.test.annotation.MockBean; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; -import junit.framework.AssertionFailedError; import lombok.SneakyThrows; import okhttp3.Call; import org.brapi.client.v2.BrAPIClient; @@ -39,23 +39,30 @@ import org.breedinginsight.dao.db.tables.daos.TraitDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.daos.cache.FetchFunction; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; +import org.breedinginsight.daos.impl.ProgramDAOImpl; import org.breedinginsight.model.ProgramBrAPIEndpoints; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIClientProvider; import org.breedinginsight.services.brapi.BrAPIClientType; +import org.breedinginsight.services.brapi.BrAPIProvider; import org.jooq.DSLContext; import org.junit.jupiter.api.*; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.redisson.api.RedissonClient; import javax.inject.Inject; import java.lang.reflect.Type; +import java.time.Duration; +import java.time.temporal.ChronoUnit; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static io.micronaut.http.HttpRequest.GET; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -72,6 +79,16 @@ public class BrAPIServiceFilterIntegrationTest extends DatabaseTest { private ProgramEntity validProgram; private TraitEntity validVariable; + @Inject + private RedissonClient redissonClient; + @Inject + private ProgramCacheProvider programCacheProvider; + + @MockBean(ProgramCacheProvider.class) + ProgramCacheProvider programCacheProvider() { + return mock(ProgramCacheProvider.class); + } + @Inject private ProgramService programService; @MockBean(ProgramService.class) @@ -81,6 +98,9 @@ public class BrAPIServiceFilterIntegrationTest extends DatabaseTest { @MockBean(BrAPIClientProvider.class) BrAPIClientProvider brAPIClientProvider() { return mock(BrAPIClientProvider.class, CALLS_REAL_METHODS); } + @Inject + private ProgramDAO programDAO; + @Inject private DSLContext dsl; @Inject @@ -91,14 +111,17 @@ public class BrAPIServiceFilterIntegrationTest extends DatabaseTest { private UserDAO userDAO; @Inject - @Client("/${micronaut.bi.api.version}") - private RxHttpClient client; + private BrAPIProvider brAPIProvider; - @AfterAll - public void finish() { - super.stopContainers(); + @MockBean(ProgramDAO.class) + ProgramDAO programDAO() { + return spy(new ProgramDAOImpl(programDao, dsl, brAPIProvider, brAPIClientProvider, Duration.of(10, ChronoUnit.MINUTES))); } + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + @BeforeAll public void setup() { @@ -106,6 +129,37 @@ public void setup() { retrieveTestData(); } + private AtomicReference> fetchException; + private AtomicReference> urlCheck; + private BrAPIClient brAPIClient; + @BeforeEach + @SneakyThrows + public void beforeEach() { + fetchException = new AtomicReference<>(Optional.empty()); + urlCheck = new AtomicReference<>(Optional.empty()); + brAPIClient = spy(new BrAPIClient()); + + when(programCacheProvider.getProgramCache(isA(FetchFunction.class), any(Class.class))).thenAnswer((Answer) invocation -> { + FetchFunction fetchFunction = invocation.getArgument(0); + return new ProgramCache(redissonClient, uuid -> { + try { + return fetchFunction.apply(uuid); + } catch (Exception e) { + fetchException.set(Optional.of(e)); + throw e; + } + }, invocation.getArgument(1)); + }); + + doAnswer(invocation -> { + BrAPIClient executingBrAPIClient = (BrAPIClient) invocation.getMock(); + // fetch the url that's set in the client + urlCheck.set(Optional.of(executingBrAPIClient.getBasePath())); + return invocation.callRealMethod(); + }).when(brAPIClient) + .execute(any(Call.class), any(Type.class)); + } + public void insertTestData() { // Insert our traits into the db var fp = FannyPack.fill("src/test/resources/sql/TraitControllerIntegrationTest.sql"); @@ -155,23 +209,21 @@ public void urlChangesForDifferentRequestsCallOne() { when(programService.getBrapiEndpoints(any(UUID.class))).thenReturn(programBrAPIEndpoints); when(programService.exists(any(UUID.class))).thenReturn(true); - // Assert our brapi url was used - CompletableFuture urlCheckFuture = checkBrAPIClientExecution(); + brAPIClient.setBasePath(coreUrl); + doReturn(brAPIClient).when(programDAO).getCoreClient(any(UUID.class)); - Flowable> call = client.exchange( + client.exchange( GET("/programs/" + validProgram.getId() + "/traits?full=true") .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); + ).blockingFirst(); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus(), "Response status is incorrect"); + assertTrue(fetchException.get().isPresent(), "No exception was thrown"); + assertTrue(fetchException.get().get() instanceof InternalServerException, "Wrong exception type was thrown"); + assertEquals("Error making BrAPI call", fetchException.get().get().getMessage()); // Check that our error is not an assertion error - assertEquals(phenoUrl, urlCheckFuture.get(), "Url was not as expected"); + assertEquals(coreUrl, urlCheck.get().get(), "URL was not as expected"); } @Test @@ -188,58 +240,26 @@ public void urlChangesForDifferentRequestsCallTwo() { when(programService.getBrapiEndpoints(any(UUID.class))).thenReturn(programBrAPIEndpoints); when(programService.exists(any(UUID.class))).thenReturn(true); - // Assert our brapi url was used - CompletableFuture urlCheckFuture = checkBrAPIClientExecution(); - - Flowable> call = client.exchange( - GET("/programs/" + validProgram.getId() + "/traits?full=true") - .contentType(MediaType.APPLICATION_JSON) - .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); - - // Expect error because brapi results will not be returned - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus(), "Response status is incorrect"); - - // Check that our error is not an assertion error - assertEquals(phenoUrl1, urlCheckFuture.get(), "Url was not as expected"); - } - - @Test - @SneakyThrows - public void getTraitsUsesFilter() { - - String phenoUrl = "http://getTraits" + UUID.randomUUID().toString() + "/brapi/v2"; - ProgramBrAPIEndpoints programBrAPIEndpoints = getBrAPIEndpoints("", phenoUrl, ""); - - reset(programService); - when(programService.getBrapiEndpoints(any(UUID.class))).thenReturn(programBrAPIEndpoints); - when(programService.exists(any(UUID.class))).thenReturn(true); - - CompletableFuture urlCheckFuture = checkBrAPIClientExecution(); + brAPIClient.setBasePath(coreUrl1); + doReturn(brAPIClient).when(programDAO).getCoreClient(any(UUID.class)); - Flowable> call = client.exchange( + // Assert our brapi url was used + client.exchange( GET("/programs/" + validProgram.getId() + "/traits?full=true") .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class - ); + ).blockingFirst(); - // Expect error because brapi results will not be returned - HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { - HttpResponse response = call.blockingFirst(); - }); - - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus(), "Response status is incorrect"); + assertTrue(fetchException.get().isPresent(), "No exception was thrown"); + assertTrue(fetchException.get().get() instanceof InternalServerException, "Wrong exception type was thrown"); + assertEquals("Error making BrAPI call", fetchException.get().get().getMessage()); - assertEquals(phenoUrl, urlCheckFuture.get(), "Url was not as expected"); + assertEquals(coreUrl1, urlCheck.get().get(), "Url was not as expected"); } @Test @SneakyThrows - public void getTraitSingleUsesFilter() { + public void getTraitSingleEditableUsesFilter() { String phenoUrl = "http://getTraitSingle" + UUID.randomUUID().toString() + "/brapi/v2"; ProgramBrAPIEndpoints programBrAPIEndpoints = getBrAPIEndpoints(null, phenoUrl, null); @@ -248,10 +268,18 @@ public void getTraitSingleUsesFilter() { when(programService.getBrapiEndpoints(any(UUID.class))).thenReturn(programBrAPIEndpoints); when(programService.exists(any(UUID.class))).thenReturn(true); - CompletableFuture urlCheckFuture = checkBrAPIClientExecution(); + validProgram.setBrapiUrl(phenoUrl); + programDAO.update(validProgram); + + when(brAPIClientProvider.getClient(BrAPIClientType.PHENO)).thenAnswer((Answer) invocation -> { + // Create a spy on our real brapi client to see what url was ultimately used + BrAPIClient realBrAPIClient = (BrAPIClient) invocation.callRealMethod(); + brAPIClient.setBasePath(realBrAPIClient.getBasePath()); + return brAPIClient; + }); Flowable> call = client.exchange( - GET("/programs/" + validProgram.getId() + "/traits/" + validVariable.getId()) + GET("/programs/" + validProgram.getId() + "/traits/" + validVariable.getId()+"/editable") .contentType(MediaType.APPLICATION_JSON) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); @@ -262,8 +290,7 @@ public void getTraitSingleUsesFilter() { }); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, e.getStatus(), "Response status is incorrect"); - - assertEquals(phenoUrl, urlCheckFuture.get(), "Url was not as expected"); + assertEquals(phenoUrl, urlCheck.get().get(), "Url was not as expected"); } public ProgramBrAPIEndpoints getBrAPIEndpoints(String coreUrl, String phenoUrl, String genoUrl){ @@ -273,40 +300,4 @@ public ProgramBrAPIEndpoints getBrAPIEndpoints(String coreUrl, String phenoUrl, .phenoUrl(phenoUrl == null ? Optional.empty() : Optional.of(phenoUrl)) .build(); } - - public CompletableFuture checkBrAPIClientExecution() throws AssertionFailedError { - - CompletableFuture future = new CompletableFuture().orTimeout(30, TimeUnit.SECONDS); - - // Takes advantage of brapi library code to mimic no return results from api. - Answer> checkBrAPIExecution = new Answer>() { - @Override - public Optional answer(InvocationOnMock invocation) throws AssertionFailedError { - BrAPIClient executingBrAPIClient = (BrAPIClient) invocation.getMock(); - // Check that our url is correct - future.complete(executingBrAPIClient.getBasePath()); - return Optional.empty(); - } - }; - - // Test the correct url was set for our provider - reset(brAPIClientProvider); - when(brAPIClientProvider.getClient(BrAPIClientType.PHENO)).thenAnswer(new Answer() { - @Override - public BrAPIClient answer(InvocationOnMock invocation) throws Throwable { - // Create a spy on our real brapi client to see what url was ultimately used - BrAPIClient realBrAPIClient = (BrAPIClient) invocation.callRealMethod(); - BrAPIClient brAPIClientSpy = spy(realBrAPIClient); - doAnswer(checkBrAPIExecution) - .when(brAPIClientSpy) - .execute(any(Call.class), any(Type.class)); - - return brAPIClientSpy; - } - }); - - return future; - - } - } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java index 3ec8d5215..2d56ffb3b 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/metadata/MetadataFilterIntegrationTest.java @@ -91,11 +91,6 @@ public void setup() { dsl.execute(fp.get("InsertSystemRoleAdmin"), otherTestUser.getId().toString()); } - @AfterAll - public void finish() { - super.stopContainers(); - } - @Test public void getSingleResponseNoMetadataSuccess() { diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java index 13add58e0..861bf27db 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiAuthorizeControllerIntegrationTest.java @@ -60,9 +60,6 @@ public class BrapiAuthorizeControllerIntegrationTest extends DatabaseTest { private ProgramEntity validProgram; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll @SneakyThrows public void setup() { diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java index 56f6d00d6..4da980a4e 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java @@ -35,7 +35,7 @@ import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.daos.ProgramDao; -import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; import org.breedinginsight.services.SpeciesService; @@ -43,11 +43,11 @@ import org.junit.jupiter.api.*; import javax.inject.Inject; - import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Comparator; import java.util.Iterator; import java.util.List; -import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -92,9 +92,6 @@ public class BrapiObservationVariablesControllerIntegrationTest extends BrAPITes private Program validProgram; private Program otherValidProgram; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll @SneakyThrows public void setup() { @@ -255,10 +252,19 @@ void userInOneProgramWithTraits() { data = result.getAsJsonArray("data"); assertEquals(2, data.size(), "Array size should be 2"); - Iterator itr = programTraits.iterator(); + List allTraits = programTraits.stream() + .sorted(Comparator.comparing(TraitEntity::getObservationVariableName)) + .collect(Collectors.toList()); + Iterator itr = allTraits.iterator(); + List sortedTraitNames = new ArrayList<>(); for (JsonElement element : data) { - JsonObject variable = element.getAsJsonObject(); + sortedTraitNames.add(element.getAsJsonObject()); + } + sortedTraitNames.sort(Comparator.comparing(o -> o.get("observationVariableName") + .getAsString())); + + for(JsonObject variable : sortedTraitNames) { checkTraits(itr.next(), variable); } @@ -298,11 +304,18 @@ void userInMultipleProgramsWithTraits() { assertEquals(4, data.size(), "Array size should be 4"); List allTraits = Stream.concat(programTraits.stream(), otherProgramTraits.stream()) - .collect(Collectors.toList()); + .sorted(Comparator.comparing(TraitEntity::getObservationVariableName)) + .collect(Collectors.toList()); Iterator itr = allTraits.iterator(); + List sortedTraitNames = new ArrayList<>(); for (JsonElement element : data) { - JsonObject variable = element.getAsJsonObject(); + sortedTraitNames.add(element.getAsJsonObject()); + } + sortedTraitNames.sort(Comparator.comparing(o -> o.get("observationVariableName") + .getAsString())); + + for(JsonObject variable : sortedTraitNames) { checkTraits(itr.next(), variable); } } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index 4c423bfb1..a76b65c77 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -83,9 +83,6 @@ public class BrAPIV2ControllerIntegrationTest extends BrAPITest { private ProgramEntity validProgram; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll @SneakyThrows public void setup() { diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 2c714a1aa..39920cb67 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -76,9 +76,6 @@ public class GermplasmControllerIntegrationTest extends BrAPITest { @Inject private BrAPIListDAO listDAO; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll @SneakyThrows public void setup() { diff --git a/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java index 090ddafce..0e2ad94e1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ProgramCacheUnitTest.java @@ -1,35 +1,28 @@ package org.breedinginsight.brapi.v2; import io.micronaut.test.annotation.MockBean; -import junit.framework.AssertionFailedError; import lombok.SneakyThrows; -import org.brapi.client.v2.BrAPIClient; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.breedinginsight.TestUtils; +import org.breedinginsight.DatabaseTest; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapi.v2.dao.FetchFunction; -import org.breedinginsight.brapi.v2.dao.ProgramCache; -import org.breedinginsight.services.ProgramService; -import org.breedinginsight.utilities.email.EmailUtil; +import org.breedinginsight.daos.cache.ProgramCache; import org.junit.jupiter.api.*; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import javax.inject.Inject; +import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.Callable; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class ProgramCacheUnitTest { +public class ProgramCacheUnitTest extends DatabaseTest { // TODO: Tests // POSTing skips refresh if one is already queued @@ -45,9 +38,6 @@ public class ProgramCacheUnitTest { @Inject BrAPIGermplasmDAO germplasmDAO; - @BeforeAll - void setup() {} - @AfterEach @SneakyThrows void setupNextTest() { @@ -60,53 +50,60 @@ void setupNextTest() { @SneakyThrows public Map mockFetch(UUID programId, Integer sleepTime) { fetchCount += 1; + System.out.println("Starting sleep at " + LocalDateTime.now()); Thread.sleep(sleepTime); + System.out.println("Woken up at " + LocalDateTime.now()); return mockBrAPI.containsKey(programId) ? new HashMap<>(mockBrAPI.get(programId).stream().collect(Collectors.toMap(germplasm -> UUID.randomUUID().toString(), germplasm -> germplasm))) : new HashMap<>(); } @SneakyThrows - public List mockPost(UUID programId, List germplasm) { + public Map mockPost(UUID programId, List germplasm) { if (!mockBrAPI.containsKey(programId)) { mockBrAPI.put(programId, germplasm); } else { List allGermplasm = mockBrAPI.get(programId); allGermplasm.addAll(germplasm); } - return germplasm; + Map germMap = new HashMap<>(); + germplasm.forEach(brAPIGermplasm -> germMap.put(brAPIGermplasm.getGermplasmDbId(), brAPIGermplasm)); + return germMap; } @Test @SneakyThrows - public void populatedRefreshQueueSkipsRefresh() { + public void populatedRefreshQueueSkipsRefresh() throws Exception { // Make a lot of post calls and just how many times the fetch method is called - ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime)); + ProgramCache cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(id, waitTime), BrAPIGermplasm.class); UUID programId = UUID.randomUUID(); int numPost = 10; int currPost = 0; while (currPost < numPost) { List newList = new ArrayList<>(); - newList.add(new BrAPIGermplasm()); + newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString())); cache.post(programId, () -> mockPost(programId, new ArrayList<>(newList))); currPost += 1; } assertTrue(fetchCount < numPost, "A fetch call was made for every post. It shouldn't."); assertEquals(1, mockBrAPI.size(), "More than one program existed in mocked brapi db."); assertEquals(numPost, mockBrAPI.get(programId).size(), "Wrong number of germplasm in db"); + while(cache.isRefreshing(programId)) { + Thread.sleep(waitTime); + } Map cachedGermplasm = cache.get(programId); assertEquals(numPost, cachedGermplasm.size(), "Wrong number of germplasm in cache"); } @Test @SneakyThrows - public void programRefreshesSeparated() { + public void programRefreshesSeparated() throws Exception { // Make a lot of post calls on different programs to check that they don't wait for each other - ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime)); + ProgramCache cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(id, waitTime), BrAPIGermplasm.class); int numPost = 10; int currPost = 0; while (currPost < numPost) { UUID id = UUID.randomUUID(); List newList = new ArrayList<>(); - newList.add(new BrAPIGermplasm()); + newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString())); cache.post(id, () -> mockPost(id, new ArrayList<>(newList))); // This doesn't have to do with our code, our mock function is just tripping over itself trying to update the number of fetches Thread.sleep(waitTime/5); @@ -122,37 +119,48 @@ public void programRefreshesSeparated() { @Test @SneakyThrows - public void initialGetMethodWaitsForLoad() { + public void onlyOneRefreshIsQueued() throws ApiException { // Test that the get method waits for an ongoing refresh to finish when there isn't any day UUID programId = UUID.randomUUID(); - ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId)); + List newList = new ArrayList<>(); + newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString())); + mockBrAPI.put(programId, new ArrayList<>(newList)); + + ProgramCache cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(id, waitTime*3), BrAPIGermplasm.class); + cache.populate(programId); + cache.get(programId); cache.get(programId); // Our fetch method should have only been called once for the initial loading - assertEquals(1, fetchCount, "Fetch method was called on get"); + assertEquals(2, fetchCount, "Fetch method was called on every get"); } @Test @SneakyThrows - public void getMethodDoesNotWaitForRefresh() { + public void postTriggersRefresh() throws Exception { // Test that the get method does not wait for a refresh when there is data present UUID programId = UUID.randomUUID(); List newList = new ArrayList<>(); - newList.add(new BrAPIGermplasm()); + newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString())); mockBrAPI.put(programId, new ArrayList<>(newList)); - ProgramCache cache = new ProgramCache<>((UUID id) -> mockFetch(id, waitTime), List.of(programId)); - Callable> postFunction = () -> mockPost(programId, new ArrayList<>(newList)); + ProgramCache cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(id, waitTime), BrAPIGermplasm.class); + cache.populate(programId); + Callable> postFunction = () -> mockPost(programId, new ArrayList<>(newList)); // Get waits for initial fetch Map cachedGermplasm = cache.get(programId); assertEquals(1, cachedGermplasm.size(), "Initial germplasm not as expected"); // Now post another object and call get immediately to see that it returns the old data + cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(id, waitTime*4), BrAPIGermplasm.class); cache.post(programId, postFunction); + System.out.println("calling get at: "+ LocalDateTime.now()); cachedGermplasm = cache.get(programId); - assertEquals(1, cachedGermplasm.size(), "Get method seemed to have waited for refresh method"); + assertEquals(2, cachedGermplasm.size(), "Get post method didn't insert the new data"); + Thread.sleep(waitTime); + assertEquals(true, cache.isRefreshing(programId), "Cache is not refreshing"); // Now wait for the fetch after the post to finish - Thread.sleep(waitTime*2); + Thread.sleep(waitTime*5); cachedGermplasm = cache.get(programId); assertEquals(2, cachedGermplasm.size(), "Get method did not get updated germplasm"); } @@ -160,39 +168,38 @@ public void getMethodDoesNotWaitForRefresh() { @Test @SneakyThrows @Order(1) - public void refreshErrorInvalidatesCache() { + public void refreshErrorInvalidatesCache() throws Exception { // Tests that data is invalidated when a refresh method fails // Tests that data is no longer retrievable when invalidated and needs to be refreshed // Set starter data UUID programId = UUID.randomUUID(); List newList = new ArrayList<>(); - newList.add(new BrAPIGermplasm()); + newList.add(new BrAPIGermplasm().germplasmDbId(UUID.randomUUID().toString())); mockBrAPI.put(programId, new ArrayList<>(newList)); - // Mock our method - ProgramCacheUnitTest mockTest = spy(this); - // Start cache - ProgramCache cache = new ProgramCache<>((UUID id) -> mockTest.mockFetch(programId, waitTime), List.of(programId)); + ProgramCache cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(programId, waitTime), BrAPIGermplasm.class); + cache.populate(programId); + + //wait for the initial fetch to complete + Thread.sleep(waitTime*2); - // Get waits for initial fetch Map cachedGermplasm = cache.get(programId); assertEquals(1, cachedGermplasm.size(), "Initial germplasm not as expected"); // Change our fetch method to throw an error now - when(mockTest.mockFetch(any(UUID.class), any(Integer.class))).thenAnswer(invocation -> {throw new ApiException("Uhoh");}); + cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> {throw new ApiException("UhOh");}, BrAPIGermplasm.class); cache.post(programId, () -> mockPost(programId, new ArrayList<>(newList))); // Give it a second so we can wait for the cache to be invalidated and refresh to finish Thread.sleep(waitTime*2); // Check that the fetch function needs to be called again since the cache was invalidated - reset(mockTest); - verify(mockTest, times(0)).mockFetch(any(UUID.class), any(Integer.class)); + cache = new ProgramCache<>(super.getRedisConnection(), (UUID id) -> mockFetch(programId, waitTime), BrAPIGermplasm.class); + assertEquals(1, fetchCount); cachedGermplasm = cache.get(programId); Thread.sleep(waitTime*2); - verify(mockTest, times(1)).mockFetch(any(UUID.class), any(Integer.class)); + assertEquals(2, fetchCount); assertEquals(2, cachedGermplasm.size(), "Newly retrieved germplasm not as expected"); } - } diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java index 19269ce95..591f2b663 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmTemplateMap.java @@ -75,11 +75,6 @@ public class GermplasmTemplateMap extends BrAPITest { @Client("/${micronaut.bi.api.version}") RxHttpClient client; - @AfterAll - public void finish() { - super.stopContainers(); - } - private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); diff --git a/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java b/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java index c23f8c610..a298361fa 100644 --- a/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java +++ b/src/test/java/org/breedinginsight/daos/DSLTransactionResultIntegrationTest.java @@ -58,9 +58,6 @@ public class DSLTransactionResultIntegrationTest extends DatabaseTest { private User actingUser; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll void setup() throws Exception { Optional userOptional = userService.getByOrcid(TestTokenValidator.TEST_USER_ORCID); diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index 758c613c3..a68d03efc 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -1,8 +1,6 @@ package org.breedinginsight.services; import com.google.gson.JsonObject; -import io.micronaut.context.annotation.Property; -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.functions.Function; import io.reactivex.functions.Function3; import lombok.SneakyThrows; @@ -11,12 +9,14 @@ import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; +import org.breedinginsight.DatabaseTest; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.services.parsers.germplasm.GermplasmFileColumns; @@ -25,7 +25,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; -import org.mockito.MockedStatic; import tech.tablesaw.api.Table; import java.io.InputStream; @@ -40,7 +39,7 @@ import static org.mockito.Mockito.*; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class BrAPIGermplasmServiceUnitTest { +public class BrAPIGermplasmServiceUnitTest extends DatabaseTest { private BrAPIListDAO listDAO; private BrAPIGermplasmDAO germplasmDAO; private ProgramService programService; @@ -48,6 +47,7 @@ public class BrAPIGermplasmServiceUnitTest { private BrAPIGermplasmService germplasmService; private BrAPIDAOUtil brAPIDAOUtil; private String referenceSource; + private ProgramCacheProvider cacheProvider; @SneakyThrows @BeforeEach @@ -57,7 +57,8 @@ void setup() { listDAO = mock(BrAPIListDAO.class); programDAO = mock(ProgramDAO.class); brAPIDAOUtil = mock(BrAPIDAOUtil.class); - germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class), brAPIDAOUtil); + cacheProvider = new ProgramCacheProvider(super.getRedisConnection()); + germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class), brAPIDAOUtil, cacheProvider); programService = mock(ProgramService.class); Field externalReferenceSource = BrAPIGermplasmDAO.class.getDeclaredField("referenceSource"); diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java index 4b78fb6c3..d5c3dc7d6 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java @@ -50,9 +50,6 @@ public class TraitValidatorIntegrationTest extends DatabaseTest { @Inject private TraitValidatorService traitValidator; - @AfterAll - public void finish() { super.stopContainers(); } - @BeforeAll public void setup() { diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index b61270a74..38af41ae3 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -75,9 +75,6 @@ public class ResponseUtilsIntegrationTest extends DatabaseTest { @Inject private UserDAO userDAO; - @AfterAll - public void finish() { super.stopContainers(); } - // Set up program locations @BeforeAll @SneakyThrows diff --git a/src/test/resources/sql/TraitControllerIntegrationTest.sql b/src/test/resources/sql/TraitControllerIntegrationTest.sql index 914e3b48d..27ac3413e 100644 --- a/src/test/resources/sql/TraitControllerIntegrationTest.sql +++ b/src/test/resources/sql/TraitControllerIntegrationTest.sql @@ -19,35 +19,35 @@ -- name: InsertProgram insert into program (species_id, name, abbreviation, documentation_url, objective, created_by, updated_by) select species.id, 'Test Program', 'test', 'localhost:8080', 'To test things', bi_user.id, bi_user.id from species -join bi_user on bi_user.name = 'system' limit 1 +join bi_user on bi_user.name = 'system' limit 1; -- name: InsertProgramNotBrapi insert into program (species_id, name, abbreviation, documentation_url, objective, created_by, updated_by) select species.id, 'Test Program Not Brapi', 'testnotbrapi', 'localhost:8080', 'To test things', bi_user.id, bi_user.id from species -join bi_user on bi_user.name = 'system' limit 1 +join bi_user on bi_user.name = 'system' limit 1; -- name: InsertProgramObservationLevel insert into program_observation_level(program_id, name, created_by, updated_by) select program.id, 'Plant', bi_user.id, bi_user.id from program -join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 +join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1; -- name: InsertProgramOntology insert into program_ontology (program_id, created_by, updated_by) select program.id, bi_user.id, bi_user.id from program -join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1 +join bi_user on bi_user.name = 'system' and program.name = 'Test Program' limit 1; -- name: InsertMethod insert into method (program_ontology_id, created_by, updated_by) select program_ontology.id, bi_user.id, bi_user.id from program_ontology join program on program.id = program_ontology.program_id and program.name = 'Test Program' -join bi_user on bi_user.name = 'system' limit 1 +join bi_user on bi_user.name = 'system' limit 1; -- name: InsertScale insert into scale (program_ontology_id, scale_name, data_type, created_by, updated_by) select program_ontology.id, 'Test Scale', 'TEXT', bi_user.id, bi_user.id from program_ontology join program on program.id = program_ontology.program_id and program.name = 'Test Program' -join bi_user on bi_user.name = 'system' limit 1 +join bi_user on bi_user.name = 'system' limit 1; -- name: InsertTrait insert into trait (program_ontology_id, observation_variable_name, method_id, scale_id, program_observation_level_id, created_by, updated_by) @@ -57,7 +57,7 @@ join program on program.id = program_ontology.program_id and program.name = 'Tes join method on method.program_ontology_id = program_ontology.id join scale on scale.program_ontology_id = program_ontology.id and scale.scale_name = 'Test Scale' join program_observation_level on program_ontology.program_id = program_observation_level.program_id and program_observation_level.name = 'Plant' -join bi_user on bi_user.name = 'system' limit 1 +join bi_user on bi_user.name = 'system' limit 1; -- name: DeleteTrait delete from trait; @@ -67,17 +67,17 @@ delete from scale; -- name: InsertOtherProgram insert into program (species_id, name, created_by, updated_by) select species.id, 'Other Test Program', bi_user.id, bi_user.id from species -join bi_user on bi_user.name = 'system' limit 1 +join bi_user on bi_user.name = 'system' limit 1; -- name: InsertOtherProgramObservationLevel insert into program_observation_level(program_id, name, created_by, updated_by) select program.id, 'Plant', bi_user.id, bi_user.id from program -join bi_user on bi_user.name = 'system' and program.name = 'Other Test Program' limit 1 +join bi_user on bi_user.name = 'system' and program.name = 'Other Test Program' limit 1; -- name: InsertOtherProgramOntology insert into program_ontology (program_id, created_by, updated_by) select program.id, bi_user.id, bi_user.id from program -join bi_user on bi_user.name = 'system' and program.name = 'Other Test Program' limit 1 +join bi_user on bi_user.name = 'system' and program.name = 'Other Test Program' limit 1; -- name: InsertManyTraits DO $$