BI-1555 - Cache program ontology terms

docker-compose.yml

@@ -30,6 +30,8 @@ services:
     depends_on:
       - bidb
       - brapi-server
+      - redis
+      - mailhog
     volumes:
       - /usr/bin/docker:/usr/bin/docker
       - /var/run/docker.sock:/var/run/docker.sock
@@ -51,6 +53,9 @@ services:
       - EMAIL_RELAY_HOST=${EMAIL_RELAY_HOST}
       - EMAIL_RELAY_PORT=${EMAIL_RELAY_PORT}
       - EMAIL_FROM=${EMAIL_FROM}
+      - REDIS_URL=${REDIS_URL:-redis://redis:6379}
+      - REDIS_TIMEOUT=${REDIS_TIMEOUT:-30s}
+      - REDIS_SSL=${REDIS_SSL:-false}
     ports:
       - ${API_INTERNAL_PORT}:${API_INTERNAL_PORT}
     networks:
@@ -62,15 +67,15 @@ services:
       - POSTGRES_DB=${DB_NAME}
       - POSTGRES_PASSWORD=${DB_PASSWORD}
     ports:
-      - 5432:5432
+      - "5432:5432"
     volumes:
       - biapi_data:/var/lib/postgresql/data
     networks:
       backend:
         aliases:
           - dbserver
   brapi-server:
-    image: breedinginsight/brapi-java-server:latest
+    image: breedinginsight/brapi-java-server:develop
     container_name: brapi-server
     depends_on:
       - bidb
@@ -93,8 +98,16 @@ services:
     container_name: mailhog
     restart: always
     ports:
-      - 1025:1025
-      - 8025:8025
+      - "1025:1025"
+      - "8025:8025"
+  redis:
+    image: redis
+    container_name: redis
+    restart: always
+    ports:
+      - "6379:6379"
+    networks:
+      backend:
 
 networks:
   backend:

pom.xml

@@ -59,9 +59,9 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <exec.mainClass>org.breedinginsight.api.Application</exec.mainClass>
-        <maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version> <!-- 2.22.2 -->
-        <maven-failsafe-plugin.version>2.19.1</maven-failsafe-plugin.version>
-        <jvm.options>--enable-preview</jvm.options>
+        <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> <!-- 2.22.2 -->
+        <maven-failsafe-plugin.version>2.22.2</maven-failsafe-plugin.version>
+        <jvm.options>-Xmx1024m -XX:MaxPermSize=256m --enable-preview</jvm.options>
 
         <jooq.version>3.16.3</jooq.version>
         <postgres.version>42.3.2</postgres.version>
@@ -395,6 +395,11 @@
             <artifactId>cloning</artifactId>
             <version>${cloning.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.redisson</groupId>
+            <artifactId>redisson-micronaut-20</artifactId>
+            <version>3.17.5</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -571,21 +576,21 @@
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-surefire-plugin</artifactId>
                     <version>${maven-surefire-plugin.version}</version>
-                    <dependencies>
-                        <dependency>
-                            <groupId>org.junit.platform</groupId>
-                            <artifactId>junit-platform-surefire-provider</artifactId>
-                            <version>1.1.0</version>
-                        </dependency>
-                        <dependency>
-                            <groupId>org.junit.jupiter</groupId>
-                            <artifactId>junit-jupiter-engine</artifactId>
-                            <version>5.7.0</version>
-                        </dependency>
-                    </dependencies>
+<!--                    <dependencies>-->
+<!--                        <dependency>-->
+<!--                            <groupId>org.junit.platform</groupId>-->
+<!--                            <artifactId>junit-platform-surefire-provider</artifactId>-->
+<!--                            <version>1.3.2</version>-->
+<!--                        </dependency>-->
+<!--                        <dependency>-->
+<!--                            <groupId>org.junit.jupiter</groupId>-->
+<!--                            <artifactId>junit-jupiter-engine</artifactId>-->
+<!--                            <version>5.9.0</version>-->
+<!--                        </dependency>-->
+<!--                    </dependencies>-->
                     <configuration>
                         <redirectTestOutputToFile>false</redirectTestOutputToFile>
-                            <argLine>${jvm.options}</argLine>
+                        <argLine>${jvm.options}</argLine>
                         <detail>true</detail>
                         <includes>
                             <include>%regex[.*]</include>

src/main/java/org/breedinginsight/brapi/v1/services/BrapiObservationVariableService.java

@@ -17,6 +17,7 @@
 package org.breedinginsight.brapi.v1.services;
 
 import lombok.extern.slf4j.Slf4j;
+import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories;
 import org.breedinginsight.api.auth.AuthenticatedUser;
 import org.breedinginsight.brapi.v1.model.*;
 import org.breedinginsight.dao.db.enums.DataType;
@@ -111,8 +112,15 @@ private ObservationVariable mapBiTraitToBrapiV1ObservationVariable(org.breedingi
 
         TraitDataType dataType = mapBiDataTypeToBrapiV1TraitDataType(biScale.getDataType());
 
-        List<String> categories = biScale.getCategories().stream()
-                .map(category -> category.getValue()).collect(Collectors.toList());
+
+        List<String> categories = new ArrayList<>();
+
+        if(biScale.getCategories() != null) {
+            categories = biScale.getCategories()
+                                .stream()
+                                .map(BrAPIScaleValidValuesCategories::getValue)
+                                .collect(Collectors.toList());
+        }
 
         ValidValues validValues = ValidValues.builder()
                 .categories(categories)
@@ -169,7 +177,7 @@ private ObservationVariable mapBiTraitToBrapiV1ObservationVariable(org.breedingi
                 //.scientist() // missing from bi trait model but stored in BrAPI service
                 //.status(trait.getStatus())
                 .submissionTimestamp(trait.getCreatedAt())
-                .synonyms(trait.getSynonyms().isEmpty() ? synonyms : trait.getSynonyms()) // TODO: fix need to have synonym for field book bug
+                .synonyms(trait.getSynonyms() == null || trait.getSynonyms().isEmpty() ? synonyms : trait.getSynonyms()) // TODO: fix need to have synonym for field book bug
                 .trait(brapiTrait)
                 .xref(trait.getId().toString())
                 .observationVariableDbId(trait.getId().toString())

src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java

@@ -27,6 +27,7 @@
 import org.breedinginsight.brapps.importer.model.exports.FileType;
 import org.breedinginsight.model.DownloadFile;
 import org.breedinginsight.services.exceptions.DoesNotExistException;
+import org.breedinginsight.utilities.Utilities;
 import org.breedinginsight.utilities.response.ResponseUtils;
 
 import javax.inject.Inject;

src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java

@@ -21,23 +21,28 @@
 import io.micronaut.context.annotation.Context;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.http.server.exceptions.InternalServerException;
+import io.micronaut.scheduling.annotation.Scheduled;
 import lombok.extern.slf4j.Slf4j;
+import org.brapi.client.v2.ApiResponse;
 import org.brapi.client.v2.model.exceptions.ApiException;
+import org.brapi.client.v2.model.queryParams.germplasm.GermplasmQueryParams;
 import org.brapi.client.v2.modules.germplasm.GermplasmApi;
 import org.brapi.v2.model.BrAPIExternalReference;
 import org.brapi.v2.model.germ.BrAPIGermplasm;
 import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms;
 import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest;
+import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse;
 import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
 import org.breedinginsight.brapps.importer.daos.ImportDAO;
 import org.breedinginsight.brapps.importer.model.ImportUpload;
 import org.breedinginsight.daos.ProgramDAO;
+import org.breedinginsight.daos.cache.ProgramCache;
+import org.breedinginsight.daos.cache.ProgramCacheProvider;
 import org.breedinginsight.model.Program;
 import org.breedinginsight.services.exceptions.DoesNotExistException;
 import org.breedinginsight.utilities.BrAPIDAOUtil;
 import org.breedinginsight.utilities.Utilities;
 
-import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.util.*;
@@ -56,20 +61,30 @@ public class BrAPIGermplasmDAO {
     @Property(name = "brapi.server.reference-source")
     private String referenceSource;
 
-    ProgramCache<String, BrAPIGermplasm> programGermplasmCache;
+    @Property(name = "micronaut.bi.api.run-scheduled-tasks")
+    private boolean runScheduledTasks;
+
+    private final ProgramCache<BrAPIGermplasm> programGermplasmCache;
 
     @Inject
-    public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil) {
+    public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramCacheProvider programCacheProvider) {
         this.programDAO = programDAO;
         this.importDAO = importDAO;
         this.brAPIDAOUtil = brAPIDAOUtil;
+        this.programGermplasmCache = programCacheProvider.getProgramCache(this::fetchProgramGermplasm, BrAPIGermplasm.class);
     }
 
-    @PostConstruct
-    private void setup() {
+    @Scheduled(initialDelay = "2s")
+    public void setup() {
+        if(!runScheduledTasks) {
+            return;
+        }
         // Populate germplasm cache for all programs on startup
-        List<UUID> programs = programDAO.getAll().stream().filter(Program::getActive).map(Program::getId).collect(Collectors.toList());
-        programGermplasmCache = new ProgramCache<>(this::fetchProgramGermplasm, programs);
+        log.debug("populating germplasm cache");
+        List<Program> programs = programDAO.getActive();
+        if(programs != null) {
+            programGermplasmCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList()));
+        }
     }
 
     /**
@@ -111,10 +126,11 @@ public List<BrAPIGermplasm> getRawGermplasm(UUID programId) throws ApiException
      */
     private Map<String, BrAPIGermplasm> fetchProgramGermplasm(UUID programId) throws ApiException {
         GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId));
-
         // Get the program key
         List<Program> programs = programDAO.get(programId);
-        if (programs.size() != 1) throw new InternalServerException("Program was not found for given key");
+        if (programs.size() != 1) {
+            throw new InternalServerException("Program was not found for given key");
+        }
         Program program = programs.get(0);
 
         // Set query params and make call
@@ -137,7 +153,7 @@ private Map<String, BrAPIGermplasm> fetchProgramGermplasm(UUID programId) throws
     private Map<String,BrAPIGermplasm> processGermplasmForDisplay(List<BrAPIGermplasm> programGermplasm, String programKey) {
         // Process the germplasm
         Map<String, BrAPIGermplasm> programGermplasmMap = new HashMap<>();
-        log.debug("processing germ for display: " + programGermplasm);
+        log.trace("processing germ for display: " + programGermplasm);
         Map<String, BrAPIGermplasm> programGermplasmByFullName = new HashMap<>();
         for (BrAPIGermplasm germplasm: programGermplasm) {
             programGermplasmByFullName.put(germplasm.getGermplasmName(), germplasm);
@@ -212,13 +228,15 @@ private Map<String,BrAPIGermplasm> processGermplasmForDisplay(List<BrAPIGermplas
 
     public List<BrAPIGermplasm> importBrAPIGermplasm(List<BrAPIGermplasm> brAPIGermplasmList, UUID programId, ImportUpload upload) throws ApiException {
         GermplasmApi api = new GermplasmApi(programDAO.getCoreClient(programId));
+        var program = programDAO.fetchOneById(programId);
         try {
-            Callable<List<BrAPIGermplasm>> postFunction = () -> brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update);
+            Callable<Map<String, BrAPIGermplasm>> postFunction = () -> {
+                List<BrAPIGermplasm> postResponse = brAPIDAOUtil.post(brAPIGermplasmList, upload, api::germplasmPost, importDAO::update);
+                return processGermplasmForDisplay(postResponse, program.getKey());
+            };
             return programGermplasmCache.post(programId, postFunction);
-        } catch (ApiException e) {
-            throw e;
         } catch (Exception e) {
-            throw new InternalServerException("Unknown error has occurred: " + e.getMessage());
+            throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e);
         }
     }