BI-2047 - Pull Existing Observation Values For An Experiment Into Field-Book

src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java

@@ -28,13 +28,17 @@
 import org.brapi.client.v2.ApiResponse;
 import org.brapi.client.v2.model.exceptions.ApiException;
 import org.brapi.client.v2.modules.phenotype.ObservationsApi;
+import org.brapi.v2.model.BrAPIIndexPagination;
+import org.brapi.v2.model.BrAPIMetadata;
+import org.brapi.v2.model.BrAPIStatus;
 import org.brapi.v2.model.BrAPIWSMIMEDataTypes;
 import org.brapi.v2.model.core.BrAPIStudy;
 import org.brapi.v2.model.pheno.BrAPIObservation;
-import org.brapi.v2.model.pheno.response.BrAPIObservationTableResponse;
+import org.brapi.v2.model.pheno.response.*;
 import org.breedinginsight.api.auth.ProgramSecured;
 import org.breedinginsight.api.auth.ProgramSecuredRoleGroup;
 import org.breedinginsight.brapi.v1.controller.BrapiVersion;
+import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO;
 import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO;
 import org.breedinginsight.brapi.v2.model.request.query.ObservationQuery;
 import org.breedinginsight.daos.ProgramDAO;
@@ -47,6 +51,8 @@
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Slf4j
 @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2)
@@ -57,13 +63,20 @@ public class BrAPIObservationsController {
     private final ProgramDAO programDAO;
     private final BrAPIStudyDAO brAPIStudyDAO;
     private final BrAPIEndpointProvider brAPIEndpointProvider;
+    private final BrAPIObservationDAO observationDAO;
 
     @Inject
-    public BrAPIObservationsController(ProgramService programService, ProgramDAO programDAO, ProgramDAO programDAO1, BrAPIStudyDAO brAPIStudyDAO, BrAPIEndpointProvider brAPIEndpointProvider) {
+    public BrAPIObservationsController(ProgramService programService,
+                                       ProgramDAO programDAO,
+                                       ProgramDAO programDAO1,
+                                       BrAPIStudyDAO brAPIStudyDAO,
+                                       BrAPIEndpointProvider brAPIEndpointProvider,
+                                       BrAPIObservationDAO brAPIObservationDAO) {
         this.programService = programService;
         this.programDAO = programDAO1;
         this.brAPIStudyDAO = brAPIStudyDAO;
         this.brAPIEndpointProvider = brAPIEndpointProvider;
+        this.observationDAO = brAPIObservationDAO;
     }
 
     @Get("/observations")
@@ -93,8 +106,99 @@ public HttpResponse observationsGet(@PathVariable("programId") UUID programId,
                                         @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource,
                                         @Nullable @QueryValue("page") Integer page,
                                         @Nullable @QueryValue("pageSize") Integer pageSize) {
-        //TODO
-        return HttpResponse.notFound();
+        log.debug("observationsGet: fetching observations by filters");
+        Optional<Program> program = programService.getById(programId);
+        if(program.isEmpty()) {
+            log.warn("Program id: " + programId + " not found");
+            return HttpResponse.notFound();
+        }
+
+        try {
+
+            // TODO: BI-2506 - implement support for all query parameters.
+            List<Object> unsupportedParams = Stream.of(
+                    observationDbId,
+                    observationUnitDbId,
+                    observationVariableDbId,
+                    locationDbId,
+                    seasonDbId,
+                    observationTimeStampRangeStart,
+                    observationTimeStampRangeEnd,
+                    observationUnitLevelName,
+                    observationUnitLevelOrder,
+                    observationUnitLevelCode,
+                    observationUnitLevelRelationshipName,
+                    observationUnitLevelRelationshipOrder,
+                    observationUnitLevelRelationshipCode,
+                    observationUnitLevelRelationshipDbId,
+                    commonCropName,
+                    programDbId,
+                    trialDbId,
+                    germplasmDbId,
+                    externalReferenceID,
+                    externalReferenceId,
+                    externalReferenceSource
+            ).filter(Objects::nonNull).collect(Collectors.toList());
+
+            if (!unsupportedParams.isEmpty()) {
+                return HttpResponse.status(HttpStatus.NOT_IMPLEMENTED).body(
+                        new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR)
+                                .message("Unsupported query parameter. Only studyDbId, page, and pageSize are supported."))))
+                );
+            }
+
+            // Get a filtered list of observations.
+            List<BrAPIObservation> observations = observationDAO.getObservationsByFilters(program.get(), studyDbId);
+
+            // If page is not provided, set it to the default value 0.
+            if (page == null) page = 0;
+            // If pageSize is not provided, set it to the default value 1000.
+            if (pageSize == null) pageSize = 1000;
+
+            // Total number of records in the unpaged super set.
+            int totalCount = observations.size();
+            // The least of pageSize and totalCount, unless pageSize is zero, in which case use totalCount.
+            int requestedPageSize = pageSize > 0 ? Math.min(pageSize, totalCount) : totalCount;
+            // Integer division and round up.
+            int totalPages = totalCount / requestedPageSize + ((totalCount % requestedPageSize == 0) ? 0 : 1);
+            log.info("(Pagination) totalCount: " + totalCount + " page (0-indexed): " + page + " requestedPageSize: " + requestedPageSize + " totalPages: " + totalPages);
+
+            // Determine validity of pagination query parameters.
+            boolean pageSizeValid = pageSize > 0;
+            boolean pageValid = page >= 0 && page < totalPages;
+
+            // Only paginate if valid pagination values were sent.
+            if (pageSizeValid && pageValid) {
+                int start = page * requestedPageSize;
+                // Account for last page, which may have fewer than requestedPageSize items, or exactly requestedPageSize items.
+                int end = (page == (totalPages - 1) && totalCount % requestedPageSize != 0) ? (start + (totalCount % requestedPageSize)) : Math.min(((page + 1) * requestedPageSize), totalCount);
+                log.info("(Pagination) start " + start + " end " + end);
+                // Sort observations so that paging is consistent and coherent.
+                observations.sort(Comparator.comparing(BrAPIObservation::getObservationDbId));
+                // Paginate response.
+                observations = observations.subList(start, end);
+            } else {
+                String errorMessage = "Invalid query parameters: page, pageSize";
+                return HttpResponse.badRequest(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR)
+                        .message(errorMessage)))));
+            }
+
+            return HttpResponse.ok(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination()
+                            .currentPage(page)
+                            .totalPages(totalPages)
+                            .pageSize(observations.size())
+                            .totalCount(totalCount)))
+                    .result(new BrAPIObservationListResponseResult().data(observations)));
+
+        } catch (ApiException e) {
+            log.error(Utilities.generateApiExceptionLogMessage(e), e);
+            return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR)
+                    .message("Error fetching observations")))));
+        } catch (Exception e) {
+            log.error("Error fetching Observations", e);
+            return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR)
+                    .message("Error fetching observations")))));
+        }
     }
 
     @Get("/observations/{observationDbId}")

src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java

@@ -38,7 +38,10 @@
 import org.breedinginsight.daos.ProgramDAO;
 import org.breedinginsight.daos.cache.ProgramCacheProvider;
 import org.breedinginsight.model.Program;
+import org.breedinginsight.model.Trait;
+import org.breedinginsight.services.TraitService;
 import org.breedinginsight.services.brapi.BrAPIEndpointProvider;
+import org.breedinginsight.services.exceptions.DoesNotExistException;
 import org.breedinginsight.utilities.BrAPIDAOUtil;
 import org.breedinginsight.utilities.Utilities;
 import org.jetbrains.annotations.NotNull;
@@ -62,6 +65,7 @@ public class BrAPIObservationDAO extends BrAPICachedDAO<BrAPIObservation> {
     private final BrAPIEndpointProvider brAPIEndpointProvider;
     private final String referenceSource;
     private boolean runScheduledTasks;
+    private final TraitService traitService;
 
     @Inject
     public BrAPIObservationDAO(ProgramDAO programDAO,
@@ -71,14 +75,15 @@ public BrAPIObservationDAO(ProgramDAO programDAO,
                                BrAPIEndpointProvider brAPIEndpointProvider,
                                @Property(name = "brapi.server.reference-source") String referenceSource,
                                @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks,
-                               ProgramCacheProvider programCacheProvider) {
+                               ProgramCacheProvider programCacheProvider, TraitService traitService) {
         this.programDAO = programDAO;
         this.importDAO = importDAO;
         this.observationUnitDAO = observationUnitDAO;
         this.brAPIDAOUtil = brAPIDAOUtil;
         this.brAPIEndpointProvider = brAPIEndpointProvider;
         this.referenceSource = referenceSource;
         this.runScheduledTasks = runScheduledTasks;
+        this.traitService = traitService;
         this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservations, BrAPIObservation.class);
     }
 
@@ -242,6 +247,42 @@ public List<BrAPIObservation> getObservationsByObservationUnitsAndStudies(Collec
                 .collect(Collectors.toList());
     }
 
+    // TODO: implement other filters in BI-2506.
+    public List<BrAPIObservation> getObservationsByFilters(Program program, String studyDbId) throws ApiException, DoesNotExistException {
+
+        String studySource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES);
+        String observationUnitSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS);
+        String observationSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATIONS);
+
+        // Get all observations for the program.
+        Collection<BrAPIObservation> observations = getProgramObservations(program.getId()).values();
+        // Build a hashmap of traits for fast lookup. The key is ObservationVariableDbId, the value is the Trait Id.
+        HashMap<String, String> traitIdsByObservationVariableDbId = traitService.getIdsByObservationVariableDbIds(program.getId(), observations.stream().map(BrAPIObservation::getObservationVariableDbId).collect(Collectors.toList()));
+
+        // Lookup studyDbId.
+        return observations.stream()
+                .filter(o -> {
+                    // Short circuit if filter is null.
+                    if (studyDbId == null) return true;
+                    Optional<BrAPIExternalReference> xref = Utilities.getExternalReference(o.getExternalReferences(), studySource);
+                    return xref.filter(brAPIExternalReference -> studyDbId.equals(brAPIExternalReference.getReferenceId())).isPresent();
+                })
+                .peek(o -> {
+                    // Translate ObservationVariableDbId.
+                    o.setObservationVariableDbId(traitIdsByObservationVariableDbId.get(o.getObservationVariableDbId()));
+                    // Translate ObservationUnitDbId.
+                    o.setObservationUnitDbId(Utilities.getExternalReference(o.getExternalReferences(), observationUnitSource)
+                            .orElseThrow(() -> new RuntimeException("observationUnit xref not found on observation")).getReferenceId());
+                    // Translate ObservationId.
+                    o.setObservationDbId(Utilities.getExternalReference(o.getExternalReferences(), observationSource)
+                            .orElseThrow(() -> new RuntimeException("observation xref not found on observation")).getReferenceId());
+                    // Translate StudyDbId.
+                    o.setStudyDbId(Utilities.getExternalReference(o.getExternalReferences(), studySource)
+                            .orElseThrow(() -> new RuntimeException("study xref not found on observation")).getReferenceId());
+                    // TODO: consider translating germplasmDbId in BI-2506.
+                }).collect(Collectors.toList());
+    }
+
     @NotNull
     private ApiResponse<Pair<Optional<BrAPIObservationListResponse>, Optional<BrAPIAcceptedSearchResponse>>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException {
         ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class);

src/main/java/org/breedinginsight/services/TraitService.java

@@ -491,4 +491,29 @@ public List<Trait> getByName(UUID programId, List<String> names) throws DoesNotE
 
         return traitDAO.getTraitsByTraitName(programId, names.stream().map(name -> Trait.builder().observationVariableName(name).build()).collect(Collectors.toList()));
     }
+
+    public Trait getByObservationVariableDbId(UUID programId, String observationVariableDbId) throws DoesNotExistException {
+        if (!programService.exists(programId)) {
+            throw new DoesNotExistException("Program does not exist");
+        }
+
+        return traitDAO.getTraitsFullByProgramId(programId).stream()
+                .filter(t -> t.getObservationVariableDbId().equals(observationVariableDbId))
+                .findFirst().orElseThrow(() -> new DoesNotExistException("Trait not found for observationVariableDbId: " + observationVariableDbId));
+
+    }
+
+    public HashMap<String, String> getIdsByObservationVariableDbIds(UUID programId, List<String> observationVariableDbIds) throws DoesNotExistException {
+        if (!programService.exists(programId)) {
+            throw new DoesNotExistException("Program does not exist");
+        }
+        return traitDAO.getTraitsFullByProgramId(programId).stream()
+                .filter(t -> observationVariableDbIds.contains(t.getObservationVariableDbId()))
+                .collect(Collectors.toMap(
+                        Trait::getObservationVariableDbId,
+                        (t) -> t.getId().toString(),
+                        (existing, replacement) -> existing,
+                        HashMap::new
+                ));
+    }
 }