From 48bc5b96b80a0f21656bd13e58ce24e0a914a61b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 13 Jun 2023 12:24:57 -0400 Subject: [PATCH 01/25] [BI-1720] - initial proof of concept --- .../brapi/v2/ExperimentController.java | 71 +++++++++++++++++-- .../request/query/ExperimentExportQuery.java | 2 + .../brapi/v2/services/BrAPITrialService.java | 2 + 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 4e4a67f18..bc0fd9713 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -29,8 +29,12 @@ import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; import javax.validation.Valid; -import java.util.List; -import java.util.UUID; +import java.io.*; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Slf4j @Controller @@ -91,17 +95,72 @@ public HttpResponse> getExperimentById( @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/zip"}) public HttpResponse datasetExport( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, @QueryValue @Valid ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { + DownloadFile downloadFile; Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); - DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); + + // If multiple files are requested, make multiple requests! + List datasetFiles = new LinkedList<>(); + if (queryParams.getEnvironments() != null && queryParams.getEnvironments().split(",").length > 1) { + String[] environments = queryParams.getEnvironments().split(","); + for (String env : environments) { + // Overwrite queryParams environments to just current env. + queryParams.setEnvironments(env); + // Get file for env. + DownloadFile file = experimentService.exportObservations(program, experimentId, queryParams); + // Add to file list. + datasetFiles.add(file); + } + } else { + datasetFiles.add(experimentService.exportObservations(program, experimentId, queryParams)); + } + + // TODO: zip if more than 1 file. + if (datasetFiles.size() > 1) + { + // Build zip file name. .zip + BrAPITrial experiment = experimentService.getExperiment(program, experimentId); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String filename = String.format("%s_%s.zip", experiment.getTrialName(), timestamp); + + PipedInputStream in = new PipedInputStream(); + final PipedOutputStream out = new PipedOutputStream(in); + new Thread(() -> { + try { + ZipOutputStream zipStream = new ZipOutputStream(out); + for (DownloadFile datasetFile : datasetFiles) { + + ZipEntry entry = new ZipEntry(datasetFile.getFileName()); + zipStream.putNextEntry(entry); + // Write datasetFile to zip. + zipStream.write(datasetFile.getStreamedFile().getInputStream().readAllBytes()); + zipStream.closeEntry(); + + } + zipStream.close(); + out.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).start(); + + StreamedFile sf = new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); + downloadFile = new DownloadFile(filename, sf); + } + else { + // There's only one file, download without zipping. + downloadFile = datasetFiles.get(0); + } + HttpResponse response = HttpResponse - .ok(datasetFile.getStreamedFile()) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); + .ok(downloadFile.getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.getFileName()); return response; } catch (Exception e) { log.info(e.getMessage(), e); diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 5f60f1008..28bab6c22 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -2,6 +2,7 @@ import io.micronaut.core.annotation.Introspected; import lombok.Getter; +import lombok.Setter; import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; @@ -17,6 +18,7 @@ public class ExperimentExportQuery { private FileType fileExtension; private String dataset; + @Setter private String environments; @NotNull private boolean includeTimestamps; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index c66671355..9fccc8ada 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -240,6 +240,8 @@ private String getOUId(BrAPIObservationUnit ou) { } private String getStudyId(BrAPIStudy study) { + // HACK: avoid null reference exceptions. + if (study == null) return null; BrAPIExternalReference studyXref = Utilities.getExternalReference( study.getExternalReferences(), String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES.getName())) From f7d5a1958162f1571bf124762b72922cee8c1ca8 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:13:57 -0400 Subject: [PATCH 02/25] [BI-1720] - comments --- .../org/breedinginsight/brapi/v2/ExperimentController.java | 5 ++++- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index bc0fd9713..da434f14a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -103,7 +103,10 @@ public HttpResponse datasetExport( try { DownloadFile downloadFile; Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); - + // TODO: if a list of environmentIds are sent, return multiple files (zipped), + // else if a single environmentId is sent, return single file (CSV/Excel), + // else (if no environmentIds are sent), return a single file (CSV/Excel) including all Environments. + // If multiple files are requested, make multiple requests! List datasetFiles = new LinkedList<>(); if (queryParams.getEnvironments() != null && queryParams.getEnvironments().split(",").length > 1) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 9fccc8ada..ae4db3c46 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -193,6 +193,7 @@ public DownloadFile exportObservations( downloadFile = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); } + // TODO: Get environment name for filename rather than UUID. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); return new DownloadFile(fileName, downloadFile); From 867e4c9ed7a1c32336603fc524d36b41629c2788 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 21 Jun 2023 14:32:20 -0400 Subject: [PATCH 03/25] [BI-1720] - refactoring in progress --- .../brapi/v2/ExperimentController.java | 57 +--- .../brapi/v2/services/BrAPITrialService.java | 245 +++++++++++++----- 2 files changed, 186 insertions(+), 116 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index da434f14a..02ae1335d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -101,65 +101,12 @@ public HttpResponse datasetExport( @QueryValue @Valid ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { - DownloadFile downloadFile; Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); + // TODO: if a list of environmentIds are sent, return multiple files (zipped), // else if a single environmentId is sent, return single file (CSV/Excel), // else (if no environmentIds are sent), return a single file (CSV/Excel) including all Environments. - - // If multiple files are requested, make multiple requests! - List datasetFiles = new LinkedList<>(); - if (queryParams.getEnvironments() != null && queryParams.getEnvironments().split(",").length > 1) { - String[] environments = queryParams.getEnvironments().split(","); - for (String env : environments) { - // Overwrite queryParams environments to just current env. - queryParams.setEnvironments(env); - // Get file for env. - DownloadFile file = experimentService.exportObservations(program, experimentId, queryParams); - // Add to file list. - datasetFiles.add(file); - } - } else { - datasetFiles.add(experimentService.exportObservations(program, experimentId, queryParams)); - } - - // TODO: zip if more than 1 file. - if (datasetFiles.size() > 1) - { - // Build zip file name. .zip - BrAPITrial experiment = experimentService.getExperiment(program, experimentId); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String timestamp = formatter.format(OffsetDateTime.now()); - String filename = String.format("%s_%s.zip", experiment.getTrialName(), timestamp); - - PipedInputStream in = new PipedInputStream(); - final PipedOutputStream out = new PipedOutputStream(in); - new Thread(() -> { - try { - ZipOutputStream zipStream = new ZipOutputStream(out); - for (DownloadFile datasetFile : datasetFiles) { - - ZipEntry entry = new ZipEntry(datasetFile.getFileName()); - zipStream.putNextEntry(entry); - // Write datasetFile to zip. - zipStream.write(datasetFile.getStreamedFile().getInputStream().readAllBytes()); - zipStream.closeEntry(); - - } - zipStream.close(); - out.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }).start(); - - StreamedFile sf = new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); - downloadFile = new DownloadFile(filename, sf); - } - else { - // There's only one file, download without zipping. - downloadFile = datasetFiles.get(0); - } + DownloadFile downloadFile = experimentService.exportObservations(program, experimentId, queryParams); HttpResponse response = HttpResponse .ok(downloadFile.getStreamedFile()) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index ae4db3c46..d827a2ed2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapi.v2.services; import io.micronaut.context.annotation.Property; +import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; @@ -30,10 +31,14 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Slf4j @Singleton @@ -113,6 +118,11 @@ public DownloadFile exportObservations( new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); + String fileName; + BrAPITrial experiment = getExperiment(program, experimentId); + // make columns present in all exports + List columns = ExperimentFileColumns.getOrderedColumns(); + // get requested environments for the experiment List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); if (!requestedEnvIds.isEmpty()) { @@ -123,82 +133,187 @@ public DownloadFile exportObservations( } expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); - // get the OUs for the requested environments - List ous = new ArrayList<>(); - Map ouByOUDbId = new HashMap<>(); - try { - for (BrAPIStudy study: expStudies) { - List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); - studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); - ous.addAll(studyOUs); - } - } catch (ApiException err) { - log.error("Error fetching observation units for a study by its DbId" + - Utilities.generateApiExceptionLogMessage(err), err); - } + // -------------------------------------------------------------------------------------------------------- + // This will hold one or more files to be downloaded. + List datasetFiles = new LinkedList<>(); + + // If multiple environments are requested, make multiple files. + if (requestedEnvIds.size() > 1) { + for (BrAPIStudy study : expStudies) { + // TODO: Make file for env. + // get the OUs for the requested environments + List ous = new ArrayList<>(); + Map ouByOUDbId = new HashMap<>(); + try { + List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); + studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); + ous.addAll(studyOUs); + } catch (ApiException err) { + log.error("Error fetching observation units for a study by its DbId" + + Utilities.generateApiExceptionLogMessage(err), err); + } - // make columns present in all exports - List columns = ExperimentFileColumns.getOrderedColumns(); + // add columns for requested dataset obsvars and timestamps + if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + isDataset = true; + obsVars = getDatasetObsVars(obsDatasetId, program); + + // make additional columns in the export for each obs variable and obs variable timestamp + addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + } - // add columns for requested dataset obsvars and timestamps - BrAPITrial experiment = getExperiment(program, experimentId); - if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && - experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - isDataset = true; - obsVars = getDatasetObsVars(obsDatasetId, program); + // make export rows from any observations + if (isDataset) { + dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); + } + if (!requestedEnvIds.isEmpty()) { + dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); + } - // make additional columns in the export for each obs variable and obs variable timestamp - addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + // update rowByOUId + addBrAPIObsToRecords( + dataset, + experiment, + program, + ouByOUDbId, + studyByDbId, + rowByOUId, + params.isIncludeTimestamps(), + obsVars + ); + + // make export rows for OUs without observations + if (rowByOUId.size() < ous.size()) { + for (BrAPIObservationUnit ou: ous) { + String ouId = getOUId(ou); + if (!rowByOUId.containsKey(ouId)) { + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); + } + } + } + // write export data to requested file format + StreamedFile file; + List> exportRows = new ArrayList<>(rowByOUId.values()); + if (fileType.equals(FileType.CSV)){ + file = CSVWriter.writeToDownload(columns, exportRows, fileType); + } else { + file = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); + } + String name = makeFileName(experiment, program, study.getStudyName()) + fileType.getExtension(); + // TODO: Add to file list. + datasetFiles.add(new DownloadFile(name, file)); + } + // TODO: Zip files. + downloadFile = zipFiles(datasetFiles); + fileName = makeZipFileName(experiment); } + else { + // Make single file with data from all environments. + // get the OUs for the requested environments + List ous = new ArrayList<>(); + Map ouByOUDbId = new HashMap<>(); + try { + for (BrAPIStudy study: expStudies) { + List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); + studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); + ous.addAll(studyOUs); + } + } catch (ApiException err) { + log.error("Error fetching observation units for a study by its DbId" + + Utilities.generateApiExceptionLogMessage(err), err); + } - // make export rows from any observations - if (isDataset) { - dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); - } - if (!requestedEnvIds.isEmpty()) { - dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); - } + // add columns for requested dataset obsvars and timestamps + if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + isDataset = true; + obsVars = getDatasetObsVars(obsDatasetId, program); + + // make additional columns in the export for each obs variable and obs variable timestamp + addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + } + + // make export rows from any observations + if (isDataset) { + dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); + } + if (!requestedEnvIds.isEmpty()) { + dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); + } - // update rowByOUId - addBrAPIObsToRecords( - dataset, - experiment, - program, - ouByOUDbId, - studyByDbId, - rowByOUId, - params.isIncludeTimestamps(), - obsVars - ); - - // make export rows for OUs without observations - if (rowByOUId.size() < ous.size()) { - for (BrAPIObservationUnit ou: ous) { - String ouId = getOUId(ou); - if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); + // update rowByOUId + addBrAPIObsToRecords( + dataset, + experiment, + program, + ouByOUDbId, + studyByDbId, + rowByOUId, + params.isIncludeTimestamps(), + obsVars + ); + + // make export rows for OUs without observations + if (rowByOUId.size() < ous.size()) { + for (BrAPIObservationUnit ou: ous) { + String ouId = getOUId(ou); + if (!rowByOUId.containsKey(ouId)) { + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); + } } } - } - // write export data to requested file format - List> exportRows = new ArrayList<>(rowByOUId.values()); - if (fileType.equals(FileType.CSV)){ - downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); - } else { - downloadFile = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); + // write export data to requested file format + List> exportRows = new ArrayList<>(rowByOUId.values()); + if (fileType.equals(FileType.CSV)){ + downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); + } else { + downloadFile = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); + } + + // TODO: Get environment name for filename rather than UUID. + String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); + fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); } + // -------------------------------------------------------------------------------------------------------- - // TODO: Get environment name for filename rather than UUID. - String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); return new DownloadFile(fileName, downloadFile); } + private StreamedFile zipFiles(List files) throws IOException { + PipedInputStream in = new PipedInputStream(); + final PipedOutputStream out = new PipedOutputStream(in); + new Thread(() -> { + try { + ZipOutputStream zipStream = new ZipOutputStream(out); + // Add each file to zip. + for (DownloadFile datasetFile : files) { + + ZipEntry entry = new ZipEntry(datasetFile.getFileName()); + zipStream.putNextEntry(entry); + // Write datasetFile to zip. + zipStream.write(datasetFile.getStreamedFile().getInputStream().readAllBytes()); + zipStream.closeEntry(); + + } + zipStream.close(); + out.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).start(); + + return new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); + } + private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, @@ -395,6 +510,14 @@ private String makeFileName(BrAPITrial experiment, Program program, String envNa envName, timestamp); } + + private String makeZipFileName(BrAPITrial experiment) { + // .zip + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + return String.format("%s_%s.zip", experiment.getTrialName(), timestamp); + } + private List filterDatasetByEnvironment( List dataset, List envIds, From 809eaa78af3639d0ac34b53851538b0af41fc2f0 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 23 Jun 2023 10:42:50 -0400 Subject: [PATCH 04/25] [BI-1720] - cleaned up and fixed bugs --- .../brapi/v2/services/BrAPITrialService.java | 257 ++++++++---------- 1 file changed, 111 insertions(+), 146 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index d827a2ed2..1d082c262 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -108,21 +108,23 @@ public DownloadFile exportObservations( Program program, UUID experimentId, ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException { - StreamedFile downloadFile; + DownloadFile downloadFile; boolean isDataset = false; List dataset = new ArrayList<>(); List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); Map studyByDbId = new HashMap<>(); + Map studyDbIdByOUId = new HashMap<>(); List requestedEnvIds = StringUtils.isNotBlank(params.getEnvironments()) ? new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); - String fileName; - BrAPITrial experiment = getExperiment(program, experimentId); // make columns present in all exports List columns = ExperimentFileColumns.getOrderedColumns(); + // add columns for requested dataset obsvars and timestamps + BrAPITrial experiment = getExperiment(program, experimentId); + // get requested environments for the experiment List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); if (!requestedEnvIds.isEmpty()) { @@ -133,159 +135,118 @@ public DownloadFile exportObservations( } expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); - // -------------------------------------------------------------------------------------------------------- - // This will hold one or more files to be downloaded. - List datasetFiles = new LinkedList<>(); - - // If multiple environments are requested, make multiple files. - if (requestedEnvIds.size() > 1) { - for (BrAPIStudy study : expStudies) { - // TODO: Make file for env. - // get the OUs for the requested environments - List ous = new ArrayList<>(); - Map ouByOUDbId = new HashMap<>(); - try { - List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); - studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); - ous.addAll(studyOUs); - } catch (ApiException err) { - log.error("Error fetching observation units for a study by its DbId" + - Utilities.generateApiExceptionLogMessage(err), err); - } + // get the OUs for the requested environments + List ous = new ArrayList<>(); + Map ouByOUDbId = new HashMap<>(); + try { + for (BrAPIStudy study: expStudies) { + List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); + studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); + ous.addAll(studyOUs); + } + } catch (ApiException err) { + log.error("Error fetching observation units for a study by its DbId" + + Utilities.generateApiExceptionLogMessage(err), err); + } - // add columns for requested dataset obsvars and timestamps - if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && - experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - isDataset = true; - obsVars = getDatasetObsVars(obsDatasetId, program); - - // make additional columns in the export for each obs variable and obs variable timestamp - addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); - } + if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + isDataset = true; + obsVars = getDatasetObsVars(obsDatasetId, program); - // make export rows from any observations - if (isDataset) { - dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); - } - if (!requestedEnvIds.isEmpty()) { - dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); - } + // make additional columns in the export for each obs variable and obs variable timestamp + addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); - // update rowByOUId - addBrAPIObsToRecords( - dataset, - experiment, - program, - ouByOUDbId, - studyByDbId, - rowByOUId, - params.isIncludeTimestamps(), - obsVars - ); - - // make export rows for OUs without observations - if (rowByOUId.size() < ous.size()) { - for (BrAPIObservationUnit ou: ous) { - String ouId = getOUId(ou); - if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); - } - } - } + } - // write export data to requested file format - StreamedFile file; - List> exportRows = new ArrayList<>(rowByOUId.values()); - if (fileType.equals(FileType.CSV)){ - file = CSVWriter.writeToDownload(columns, exportRows, fileType); - } else { - file = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); - } - String name = makeFileName(experiment, program, study.getStudyName()) + fileType.getExtension(); - // TODO: Add to file list. - datasetFiles.add(new DownloadFile(name, file)); - } - // TODO: Zip files. - downloadFile = zipFiles(datasetFiles); - fileName = makeZipFileName(experiment); + // make export rows from any observations + if (isDataset) { + dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); } - else { - // Make single file with data from all environments. - // get the OUs for the requested environments - List ous = new ArrayList<>(); - Map ouByOUDbId = new HashMap<>(); - try { - for (BrAPIStudy study: expStudies) { - List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); - studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); - ous.addAll(studyOUs); + if (!requestedEnvIds.isEmpty()) { + dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); + } + + // Update rowByOUId and studyDbIdByOUId. + addBrAPIObsToRecords( + dataset, + experiment, + program, + ouByOUDbId, + studyByDbId, + rowByOUId, + params.isIncludeTimestamps(), + obsVars, + studyDbIdByOUId + ); + + + // make export rows for OUs without observations + if (rowByOUId.size() < ous.size()) { + for (BrAPIObservationUnit ou: ous) { + String ouId = getOUId(ou); + // Map Observation Unit to the Study it belongs to. + studyDbIdByOUId.put(ouId, ou.getStudyDbId()); + if (!rowByOUId.containsKey(ouId)) { + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); } - } catch (ApiException err) { - log.error("Error fetching observation units for a study by its DbId" + - Utilities.generateApiExceptionLogMessage(err), err); } + } - // add columns for requested dataset obsvars and timestamps - if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && - experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - isDataset = true; - obsVars = getDatasetObsVars(obsDatasetId, program); - - // make additional columns in the export for each obs variable and obs variable timestamp - addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + // If one or more envs requested, create a separate file for each env, then zip if there are multiple. + if (!requestedEnvIds.isEmpty()) { + // This will hold a list of rows for each study, each list will become a separate file. + Map>> rowsByStudyId = new HashMap<>(); + + for (Map row: rowByOUId.values()) { + String studyId = studyDbIdByOUId.get((String)row.get(ExperimentObservation.Columns.OBS_UNIT_ID)); + // Initialize key with empty list if it is not present. + if (!rowsByStudyId.containsKey(studyId)) + { + rowsByStudyId.put(studyId, new ArrayList>()); + } + // Add row to appropriate list in rowsByStudyId. + rowsByStudyId.get(studyId).add(row); } - - // make export rows from any observations - if (isDataset) { - dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); + List files = new ArrayList<>(); + // Generate a file for each study. + for (Map.Entry>> entry: rowsByStudyId.entrySet()) { + StreamedFile streamedFile = writeToStreamedFile(columns, entry.getValue(), fileType, "Experiment Data"); + String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); + // Add to file list. + files.add(new DownloadFile(name, streamedFile)); } - if (!requestedEnvIds.isEmpty()) { - dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); + if (files.size() == 1) { + // Don't zip, as there is a single file. + downloadFile = files.get(0); } - - // update rowByOUId - addBrAPIObsToRecords( - dataset, - experiment, - program, - ouByOUDbId, - studyByDbId, - rowByOUId, - params.isIncludeTimestamps(), - obsVars - ); - - // make export rows for OUs without observations - if (rowByOUId.size() < ous.size()) { - for (BrAPIObservationUnit ou: ous) { - String ouId = getOUId(ou); - if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); - } - } + else { + // Zip, as there are multiple files. + StreamedFile zipFile = zipFiles(files); + downloadFile = new DownloadFile(makeZipFileName(experiment), zipFile); } - - // write export data to requested file format + } else { List> exportRows = new ArrayList<>(rowByOUId.values()); - if (fileType.equals(FileType.CSV)){ - downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); - } else { - downloadFile = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); - } - - // TODO: Get environment name for filename rather than UUID. + // write export data to requested file format + StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, "Experiment Data"); + // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); + String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); + downloadFile = new DownloadFile(fileName, streamedFile); } - // -------------------------------------------------------------------------------------------------------- - return new DownloadFile(fileName, downloadFile); + return downloadFile; + } + + private StreamedFile writeToStreamedFile(List columns, List> data, FileType extension, String sheetName) throws IOException { + if (extension.equals(FileType.CSV)){ + return CSVWriter.writeToDownload(columns, data, extension); + } else { + return ExcelWriter.writeToDownload(sheetName, columns, data, extension); + } } private StreamedFile zipFiles(List files) throws IOException { @@ -322,7 +283,8 @@ private void addBrAPIObsToRecords( Map studyByDbId, Map> rowByOUId, boolean includeTimestamp, - List obsVars) throws ApiException, DoesNotExistException { + List obsVars, + Map studyDbIdByOUId) throws ApiException, DoesNotExistException { Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { @@ -344,6 +306,9 @@ private void addBrAPIObsToRecords( addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } + + // Map Observation Unit to the Study it belongs to. + studyDbIdByOUId.put(ouId, ou.getStudyDbId()); } } @@ -522,10 +487,10 @@ private List filterDatasetByEnvironment( List dataset, List envIds, Map studyByDbId) { - return dataset - .stream() - .filter(obs -> envIds.contains(getStudyId(studyByDbId.get(obs.getStudyDbId())))) - .collect(Collectors.toList()); + return dataset + .stream() + .filter(obs -> envIds.contains(getStudyId(studyByDbId.get(obs.getStudyDbId())))) + .collect(Collectors.toList()); } -} +} \ No newline at end of file From a4653dd4fd8ae635438539e7683603107400a303 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:07:11 -0400 Subject: [PATCH 05/25] [BI-1720] - expanded unit test (WIP) --- .../importer/model/exports/FileType.java | 5 +- .../java/org/breedinginsight/TestUtils.java | 46 +++++++++- .../ExperimentControllerIntegrationTest.java | 89 +++++++++++++++---- 3 files changed, 119 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/exports/FileType.java b/src/main/java/org/breedinginsight/brapps/importer/model/exports/FileType.java index f3a81ecca..2211cbbfc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/exports/FileType.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/exports/FileType.java @@ -26,8 +26,9 @@ public enum FileType { XLS("xls", ".xls", "application/vnd.ms-excel"), XLSX("xlsx", ".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), - CSV("csv", ".csv", "text/csv"); - + CSV("csv", ".csv", "text/csv"), + // NOTE: MIME type is not application/zip because Micronaut doesn't natively support it. + ZIP("zip", ".zip", "application/octet-stream"); private String name; private String extension; private String mimeType; diff --git a/src/test/java/org/breedinginsight/TestUtils.java b/src/test/java/org/breedinginsight/TestUtils.java index effb88082..3ef061ce0 100644 --- a/src/test/java/org/breedinginsight/TestUtils.java +++ b/src/test/java/org/breedinginsight/TestUtils.java @@ -31,12 +31,14 @@ import org.breedinginsight.model.Trait; import se.sawano.java.text.AlphanumericComparator; -import java.io.File; +import java.io.*; import java.time.OffsetDateTime; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import static io.micronaut.http.HttpRequest.*; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -233,4 +235,46 @@ public static void insertTestTraits(Gson gson, RxHttpClient client, Program prog HttpResponse response = call.blockingFirst(); } + + public static void unzipFile(InputStream stream, String destDirPath) throws IOException { + File destDir = new File(destDirPath); + destDir.mkdirs(); + + byte[] buffer = new byte[1024]; + ZipInputStream zis = new ZipInputStream(stream); + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + File newFile = newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + // write file content + FileOutputStream fos = new FileOutputStream(newFile); + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + fos.close(); + } + zipEntry = zis.getNextEntry(); + } + + zis.closeEntry(); + zis.close(); + } + + public static File newFile(File destDir, ZipEntry zipEntry) throws IOException { + File destFile = new File(destDir, zipEntry.getName()); + + String destDirPath = destDir.getCanonicalPath(); + String destFilePath = destFile.getCanonicalPath(); + + if (!destFilePath.startsWith(destDirPath + File.separator)) { + throw new IOException("Entry is outside of the target dir: " + zipEntry.getName()); + } + + return destFile; + } } diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index fb5f79f92..b4a3a9dd7 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -180,6 +180,7 @@ void setup() throws Exception { .get(0).getAsJsonObject() .get("trial").getAsJsonObject() .get("id").getAsString(); + // Add first environmentId. envIds.add(importResult .get("preview").getAsJsonObject() .get("rows").getAsJsonArray() @@ -190,6 +191,17 @@ void setup() throws Exception { .get(2).getAsJsonObject() .get("referenceID").getAsString() ); + // Add second environmentId. + envIds.add(importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(1).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString() + ); } /* @@ -208,18 +220,31 @@ void setup() throws Exception { - export populated dataset, multiple environment, xlsx format */ @ParameterizedTest - @CsvSource(value = {"true,false,CSV", "true,true,CSV", - "false,false,CSV", "false,true,CSV", - "true,false,XLS", "true,true,XLS", - "false,false,XLS", "false,true,XLS", - "true,false,XLSX", "true,true,XLSX", - "false,false,XLSX", "false,true,XLSX",}) + @CsvSource(value = {"true,false,CSV,1", "true,true,CSV,1", + "false,false,CSV,1", "false,true,CSV,1", + "true,false,XLS,1", "true,true,XLS,1", + "false,false,XLS,1", "false,true,XLS,1", + "true,false,XLSX,1", "true,true,XLSX,1", + "false,false,XLSX,1", "false,true,XLSX,1", + "true,false,CSV,2", "true,true,CSV,2", + "false,false,CSV,2", "false,true,CSV,2", + "true,false,XLS,2", "true,true,XLS,2", + "false,false,XLS,2", "false,true,XLS,2", + "true,false,XLSX,2", "true,true,XLSX,2", + "false,false,XLSX,2", "false,true,XLSX,2",}) @SneakyThrows - void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String extension) { + void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String extension, int numberOfEnvs) { + // TODO: move to setup and cleanup (?) methods. + // Temporary directory to extract zip into, test will clean up after use. + String tempDir = "./zip_temp_dir/"; + // If more than 1 envId is sent as a query param, a zip file is expected response. + boolean zipExpected = requestEnv && (numberOfEnvs > 1); // Download test experiment String envParam = "all=true"; if (requestEnv) { - envParam = "environments=" + String.join(",", envIds); + // Build environment query param with 1 or all envIds. + String envs = numberOfEnvs == 1 ? envIds.get(0) : String.join(",", envIds); + envParam = "environments=" + envs; } Flowable> call = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", @@ -231,25 +256,53 @@ void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String exte // Assert 200 response assertEquals(HttpStatus.OK, response.getStatus()); + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + // Assert file format fidelity Map mediaTypeByExtension = new HashMap<>(); mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); + mediaTypeByExtension.put("ZIP", FileType.ZIP.getMimeType()); String downloadMediaType = response.getHeaders().getContentType().orElseThrow(Exception::new); - assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); - - // Assert import/export fidelity and presence of observation units in export - ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); - Table download = Table.create(); - if (extension.equals("CSV")) { - download = FileUtil.parseTableFromCsv(bodyStream); + // If zip is expected, check that it is indeed a zip file, then unzip and check each file. + if (zipExpected) { + assertEquals(FileType.ZIP.getMimeType(), downloadMediaType); + // Unzip into tempDir. + TestUtils.unzipFile(bodyStream, tempDir); + + for (File file : Objects.requireNonNull(new File(tempDir).listFiles())) { + FileInputStream fileStream = new FileInputStream(file); + // TODO: potentially check each file's extension? + // TODO: DRY. + // Assert import/export fidelity and presence of observation units in export + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(fileStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(fileStream, 0); + } + // TODO: fix this method - it's expecting all data in one file. + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + } } - if (extension.equals("XLS") || extension.equals("XLSX")) { - download = FileUtil.parseTableFromExcel(bodyStream, 0); + else { + assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); + // TODO: DRY. + // Assert import/export fidelity and presence of observation units in export + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); } - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); From 2ed7b638963b06c09d22137b2039ef08eed14de1 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:49:19 -0400 Subject: [PATCH 06/25] [BI-1720] - completed unit test --- .../ExperimentControllerIntegrationTest.java | 173 ++++++++---------- 1 file changed, 75 insertions(+), 98 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index b4a3a9dd7..f8bfe3c4e 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -12,6 +12,7 @@ import io.micronaut.test.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; +import org.apache.commons.io.FileUtils; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.BrAPITest; @@ -31,6 +32,7 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.FileUtil; @@ -180,70 +182,42 @@ void setup() throws Exception { .get(0).getAsJsonObject() .get("trial").getAsJsonObject() .get("id").getAsString(); - // Add first environmentId. - envIds.add(importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceID").getAsString() - ); - // Add second environmentId. - envIds.add(importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(1).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceID").getAsString() - ); + // Add environmentIds. + envIds.add(getEnvId(importResult, 0)); + envIds.add(getEnvId(importResult, 1)); } - /* - Tests - - export empty dataset, single environment, csv format - - export empty dataset, single environment, xls format - - export empty dataset, single environment, xlsx format - - export populated dataset, single environment, csv format - - export populated dataset, single environment, xls format - - export populated dataset, single environment, xlsx format - - export empty dataset, multiple environment, csv format - - export empty dataset, multiple environment, xls format - - export empty dataset, multiple environment, xlsx format - - export populated dataset, multiple environment, csv format - - export populated dataset, multiple environment, xls format - - export populated dataset, multiple environment, xlsx format + /** + * Tests all 18 permutations of + * 3 formats: [CSV, XLS, XLSX], + * 3 env query param options: [None, 1, 2], + * 2 timestamp options [with, without]. + * @param includeTimestamps when true, timestamp columns are requested. + * @param extension the file extension requested. + * @param numberOfEnvsRequested 0 -> no env params sent, >= 1 -> requested number of envIds sent as query params. */ @ParameterizedTest - @CsvSource(value = {"true,false,CSV,1", "true,true,CSV,1", - "false,false,CSV,1", "false,true,CSV,1", - "true,false,XLS,1", "true,true,XLS,1", - "false,false,XLS,1", "false,true,XLS,1", - "true,false,XLSX,1", "true,true,XLSX,1", - "false,false,XLSX,1", "false,true,XLSX,1", - "true,false,CSV,2", "true,true,CSV,2", - "false,false,CSV,2", "false,true,CSV,2", - "true,false,XLS,2", "true,true,XLS,2", - "false,false,XLS,2", "false,true,XLS,2", - "true,false,XLSX,2", "true,true,XLSX,2", - "false,false,XLSX,2", "false,true,XLSX,2",}) + @CsvSource(value = { + "true,CSV,0", "false,CSV,0", + "true,XLS,0", "false,XLS,0", + "true,XLSX,0", "false,XLSX,0", + "true,CSV,1", "false,CSV,1", + "true,XLS,1", "false,XLS,1", + "true,XLSX,1", "false,XLSX,1", + "true,CSV,2", "false,CSV,2", + "true,XLS,2", "false,XLS,2", + "true,XLSX,2", "false,XLSX,2",}) @SneakyThrows - void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String extension, int numberOfEnvs) { - // TODO: move to setup and cleanup (?) methods. + void downloadDatasets(boolean includeTimestamps, String extension, int numberOfEnvsRequested) { // Temporary directory to extract zip into, test will clean up after use. String tempDir = "./zip_temp_dir/"; // If more than 1 envId is sent as a query param, a zip file is expected response. - boolean zipExpected = requestEnv && (numberOfEnvs > 1); + boolean zipExpected = numberOfEnvsRequested > 1; // Download test experiment String envParam = "all=true"; - if (requestEnv) { + if (numberOfEnvsRequested > 0) { // Build environment query param with 1 or all envIds. - String envs = numberOfEnvs == 1 ? envIds.get(0) : String.join(",", envIds); + String envs = numberOfEnvsRequested == 1 ? envIds.get(0) : String.join(",", envIds); envParam = "environments=" + envs; } Flowable> call = client.exchange( @@ -273,34 +247,22 @@ void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String exte for (File file : Objects.requireNonNull(new File(tempDir).listFiles())) { FileInputStream fileStream = new FileInputStream(file); - // TODO: potentially check each file's extension? - // TODO: DRY. - // Assert import/export fidelity and presence of observation units in export - Table download = Table.create(); - if (extension.equals("CSV")) { - download = FileUtil.parseTableFromCsv(fileStream); - } - if (extension.equals("XLS") || extension.equals("XLSX")) { - download = FileUtil.parseTableFromExcel(fileStream, 0); - } - // TODO: fix this method - it's expecting all data in one file. - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + // Filter rows based on env in file name. + List> filteredRows = rows.stream() + .filter(row -> file.getName().contains(row.get(ExperimentObservation.Columns.ENV).toString())) + .collect(Collectors.toList()); + parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps); } } else { assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); - // TODO: DRY. - // Assert import/export fidelity and presence of observation units in export - Table download = Table.create(); - if (extension.equals("CSV")) { - download = FileUtil.parseTableFromCsv(bodyStream); - } - if (extension.equals("XLS") || extension.equals("XLSX")) { - download = FileUtil.parseTableFromExcel(bodyStream, 0); - } - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); - } + // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. + List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; + parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps); + } + // Remove temp directory after each test run. + FileUtils.deleteDirectory(new File(tempDir)); } private File writeDataToFile(List> data, List traits) throws IOException { @@ -389,15 +351,32 @@ private List createGermplasm(int numToCreate) { return germplasm; } + + private void parseAndCheck(InputStream stream, + String extension, + boolean requestEnv, + List> rows, + boolean includeTimestamps) throws ParsingException { + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(stream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(stream, 0); + } + // Assert import/export fidelity and presence of observation units in export + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + } + private void checkDownloadTable( boolean requestEnv, - List> importRows, + List> requestedImportRows, Table table, boolean includeTimestamps, String extension) { // Filename is correct: _Observation Dataset [-]__ - List> requestedImportRows = importRows; - + List expectedEnvNames = requestedImportRows.stream() + .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()).collect(Collectors.toList()); // All columns included Integer expectedColNumber = columns.size(); if (includeTimestamps) { @@ -405,24 +384,8 @@ private void checkDownloadTable( } assertEquals(expectedColNumber, table.columnCount()); - if (!requestEnv) { - - // All environments downloaded - importRows.stream() - .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()) - .distinct() - .collect(Collectors.toList()) - .forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); - - } else { - - // Only requested environment downloaded - requestedImportRows = importRows - .stream() - .filter(row -> row.get("Env").toString().equals("Env1")).collect(Collectors.toList()); - assertEquals(1, table.stringColumn("Env").countUnique()); - assertTrue(table.stringColumn("Env").contains("Env1")); - } + // Check that requested envs are present. + expectedEnvNames.forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); List> matchingImportRows = new ArrayList<>(); Optional> matchingImportRow; @@ -451,8 +414,9 @@ private void checkDownloadTable( } assertEquals(requestedImportRows.size(),matchingImportRows.size()); - // Observation units populated + // Observation units populated. assertEquals(0, table.column("ObsUnitID").countMissing()); + // Observation Unit IDs are assigned. assertEquals(requestedImportRows.size(), table.column("ObsUnitID").countUnique()); } @@ -500,4 +464,17 @@ private boolean isMatchedRow(Map importRow, Row downloadRow) { } }).count() == importRow.size(); } + + private String getEnvId(JsonObject result, int index) { + return result + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(index).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString(); + } + } From e48da59cff53a04e6ff777f917aa844e3abafb06 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jul 2023 15:10:34 -0400 Subject: [PATCH 07/25] [BI-1720] - updated comment --- .../org/breedinginsight/brapi/v2/ExperimentController.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 02ae1335d..51711d28c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -103,9 +103,9 @@ public HttpResponse datasetExport( try { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); - // TODO: if a list of environmentIds are sent, return multiple files (zipped), - // else if a single environmentId is sent, return single file (CSV/Excel), - // else (if no environmentIds are sent), return a single file (CSV/Excel) including all Environments. + // if a list of environmentIds are sent, return multiple files (zipped), + // else if a single environmentId is sent, return single file (CSV/Excel), + // else (if no environmentIds are sent), return a single file (CSV/Excel) including all Environments. DownloadFile downloadFile = experimentService.exportObservations(program, experimentId, queryParams); HttpResponse response = HttpResponse From c6a6cbfe56e8013e011b3bd6c30051628b072879 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jul 2023 15:27:43 -0400 Subject: [PATCH 08/25] [BI-1720] - removed unused imports --- .../org/breedinginsight/brapi/v2/ExperimentController.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 51711d28c..791de3ee1 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -29,12 +29,7 @@ import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; import javax.validation.Valid; -import java.io.*; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; @Slf4j @Controller From cf113c7795eecb2b7bd23b0f6c7669c907b81b89 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jul 2023 15:59:15 -0400 Subject: [PATCH 09/25] [BI-1720] - fixed media type --- .../java/org/breedinginsight/brapi/v2/ExperimentController.java | 2 +- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 791de3ee1..c9d9f6640 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -90,7 +90,7 @@ public HttpResponse> getExperimentById( @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/zip"}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}) public HttpResponse datasetExport( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, @QueryValue @Valid ExperimentExportQuery queryParams) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 1d082c262..0126926a4 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -271,7 +271,7 @@ private StreamedFile zipFiles(List files) throws IOException { throw new RuntimeException(e); } }).start(); - + // NOTE: Micronaut doesn't define application/zip in MediaType, use application/octet-stream. return new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); } From a95e95fdda1a162416df8cd3a519fe0dd05b7fc7 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:04:39 -0400 Subject: [PATCH 10/25] [BI-1720] - cleaned up ExperimentExportQuery.java --- .../v2/model/request/query/ExperimentExportQuery.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 28bab6c22..6bcf07e22 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -2,23 +2,15 @@ import io.micronaut.core.annotation.Introspected; import lombok.Getter; -import lombok.Setter; -import org.breedinginsight.api.model.v1.request.query.FilterRequest; -import org.breedinginsight.api.model.v1.request.query.SearchRequest; -import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.brapps.importer.model.exports.FileType; -import org.jooq.tools.StringUtils; import javax.validation.constraints.NotNull; -import java.util.ArrayList; -import java.util.List; @Getter @Introspected public class ExperimentExportQuery { private FileType fileExtension; private String dataset; - @Setter private String environments; @NotNull private boolean includeTimestamps; From 4eb2c6f228b047928e46043e2137e44144115c31 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:11:19 -0400 Subject: [PATCH 11/25] [BI-1720] - simplified TestUtils::unzipFile --- src/test/java/org/breedinginsight/TestUtils.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/test/java/org/breedinginsight/TestUtils.java b/src/test/java/org/breedinginsight/TestUtils.java index 3ef061ce0..91bdd5e04 100644 --- a/src/test/java/org/breedinginsight/TestUtils.java +++ b/src/test/java/org/breedinginsight/TestUtils.java @@ -244,7 +244,7 @@ public static void unzipFile(InputStream stream, String destDirPath) throws IOEx ZipInputStream zis = new ZipInputStream(stream); ZipEntry zipEntry = zis.getNextEntry(); while (zipEntry != null) { - File newFile = newFile(destDir, zipEntry); + File newFile = new File(destDir, zipEntry.getName()); if (zipEntry.isDirectory()) { if (!newFile.isDirectory() && !newFile.mkdirs()) { throw new IOException("Failed to create directory " + newFile); @@ -264,17 +264,4 @@ public static void unzipFile(InputStream stream, String destDirPath) throws IOEx zis.closeEntry(); zis.close(); } - - public static File newFile(File destDir, ZipEntry zipEntry) throws IOException { - File destFile = new File(destDir, zipEntry.getName()); - - String destDirPath = destDir.getCanonicalPath(); - String destFilePath = destFile.getCanonicalPath(); - - if (!destFilePath.startsWith(destDirPath + File.separator)) { - throw new IOException("Entry is outside of the target dir: " + zipEntry.getName()); - } - - return destFile; - } } From 65675f713e7d31d7194a7dcc90b50ce287580b63 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Sun, 16 Jul 2023 16:13:37 -0400 Subject: [PATCH 12/25] [BI-1720] - added BrAPI Study specific code --- .../brapi/v2/StudyController.java | 107 ++++++++++ .../brapi/v2/dao/BrAPIStudyDAO.java | 198 ++++++++++++++++++ .../v2/model/request/query/StudyQuery.java | 119 +++++++++++ .../brapi/v2/services/BrAPIStudyService.java | 59 ++++++ .../response/mappers/StudyQueryMapper.java | 54 +++++ 5 files changed, 537 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/StudyController.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java create mode 100644 src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java new file mode 100644 index 000000000..69e5697a5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java @@ -0,0 +1,107 @@ +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.*; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.api.model.v1.response.DataResponse; +import org.breedinginsight.api.model.v1.response.Response; +import org.breedinginsight.api.model.v1.validators.QueryValid; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; +import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; +import org.breedinginsight.brapi.v2.services.BrAPIStudyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.response.ResponseUtils; +import org.breedinginsight.utilities.response.mappers.StudyQueryMapper; + +import javax.inject.Inject; +import javax.validation.Valid; +import java.util.List; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}") +@Secured(SecurityRule.IS_AUTHENTICATED) +public class StudyController { + + private final BrAPIStudyService studyService; + private final StudyQueryMapper studyQueryMapper; + + + @Inject + public StudyController(BrAPIStudyService studyService, StudyQueryMapper studyQueryMapper) { + this.studyService = studyService; + this.studyQueryMapper = studyQueryMapper; + } + + @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/search/studies{?queryParams*}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse>>> searchStudy( + @PathVariable("programId") UUID programId, + @QueryValue @QueryValid(using = StudyQueryMapper.class) @Valid BrapiQuery queryParams, + @Body @Valid StudyQuery body) { + try { + log.debug("fetching studies for program: " + programId); + List studies = studyService.getStudies(programId); + queryParams.setSortField(studyQueryMapper.getDefaultSortField()); + queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder()); + // Filter the response. + return ResponseUtils.getBrapiQueryResponse(studies, studyQueryMapper, queryParams, body.constructSearchRequest()); + } catch (ApiException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); + } + } + + @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/studies{?queryParams*}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse>>> getStudy( + @PathVariable("programId") UUID programId, + @QueryValue @QueryValid(using = StudyQueryMapper.class) @Valid StudyQuery queryParams) { + try { + log.debug("fetching studies for program: " + programId); + + List study = studyService.getStudies(programId); + SearchRequest searchRequest = queryParams.constructSearchRequest(); + return ResponseUtils.getBrapiQueryResponse(study, studyQueryMapper, queryParams, searchRequest); + } catch (ApiException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); + } catch (IllegalArgumentException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); + } + } + + @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/studies/{studyId}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse> getSingleStudy( + @PathVariable("programId") UUID programId, + @PathVariable("studyId") String studyId) { + try { + log.debug("fetching study id:" + studyId +" for program: " + programId); + Response response = new Response(studyService.getStudyByUUID(programId, studyId)); + return HttpResponse.ok(response); + } catch (InternalServerException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); + } catch (DoesNotExistException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.NOT_FOUND, "Study not found"); + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java new file mode 100644 index 000000000..53ec5d5b0 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -0,0 +1,198 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; + +import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Context; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.core.StudiesApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +@Context +public class BrAPIStudyDAO { + + private final ProgramDAO programDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + + @Property(name = "brapi.server.reference-source") + private String referenceSource; + + @Property(name = "micronaut.bi.api.run-scheduled-tasks") + private boolean runScheduledTasks; + + private final ProgramCache programStudyCache; + + private final BrAPIEndpointProvider brAPIEndpointProvider; + + @Inject + public BrAPIStudyDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramCacheProvider programCacheProvider, BrAPIEndpointProvider brAPIEndpointProvider) { + this.programDAO = programDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.programStudyCache = programCacheProvider.getProgramCache(this::fetchProgramStudy, BrAPIStudy.class); + this.brAPIEndpointProvider = brAPIEndpointProvider; + } + + @Scheduled(initialDelay = "2s") + public void setup() { + if(!runScheduledTasks) { + return; + } + // Populate study cache for all programs on startup + log.debug("populating study cache"); + List programs = programDAO.getActive(); + if(programs != null) { + programStudyCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + /** + * Fetch the study for this program, and process it to remove storage specific values + * @param programId + * @return this program's study + * @throws ApiException + */ + public List getStudies(UUID programId) throws ApiException { + return new ArrayList<>(programStudyCache.get(programId).values()); + } + + /** + * Fetch formatted study for this program + * @param programId + * @return Map + * @throws ApiException + */ + private Map fetchProgramStudy(UUID programId) throws ApiException { + StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); + // Get the program key + List programs = programDAO.get(programId); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given key"); + } + Program program = programs.get(0); + + // Set query params and make call + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); + studySearch.externalReferenceIDs(List.of(programId.toString())); + studySearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); + return processStudyForDisplay(brAPIDAOUtil.search( + api::searchStudiesPost, + api::searchStudiesSearchResultsDbIdGet, + studySearch + ), program.getKey()); + } + + /** + * Process study into a format for display + * @param programStudy + * @return Map + * @throws ApiException + */ + private Map processStudyForDisplay(List programStudy, String programKey) { + // Process the study + Map programStudyMap = new HashMap<>(); + log.trace("processing germ for display: " + programStudy); + Map programStudyByFullName = new HashMap<>(); + for (BrAPIStudy study: programStudy) { + programStudyByFullName.put(study.getStudyName(), study); + // Remove program key from studyName, trialName and locationName. + if (study.getStudyName() != null) { + // Study name is appended with program key and experiment sequence number, need to remove. + study.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey)); + } + if (study.getTrialName() != null) { + study.setTrialName(Utilities.removeProgramKey(study.getTrialName(), programKey)); + } + if (study.getLocationName() != null) { + study.setLocationName(Utilities.removeProgramKey(study.getLocationName(), programKey)); + } + } + + String refSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); + // Add to map. + for (BrAPIStudy study: programStudy) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if(additionalInfo == null) { + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + + BrAPIExternalReference extRef = study.getExternalReferences().stream() + .filter(reference -> reference.getReferenceSource().equals(refSource)) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + String studyId = extRef.getReferenceID(); + programStudyMap.put(studyId, study); + } + + return programStudyMap; + } + + public BrAPIStudy getStudyByUUID(String studyId, UUID programId) throws ApiException, DoesNotExistException { + Map cache = programStudyCache.get(programId); + BrAPIStudy study = null; + if (cache != null) { + study = cache.get(studyId); + } + if (study == null) { + throw new DoesNotExistException("UUID for this study does not exist"); + } + return study; + } + + public Optional getStudyByDBID(String studyDbId, UUID programId) throws ApiException { + Map cache = programStudyCache.get(programId); + //key is UUID, want to filter by DBID + BrAPIStudy study = null; + if (cache != null) { + study = cache.values().stream().filter(x -> x.getStudyDbId().equals(studyDbId)).collect(Collectors.toList()).get(0); + } + return Optional.ofNullable(study); + } + + public List getStudiesByDBID(Collection studyDbIds, UUID programId) throws ApiException { + Map cache = programStudyCache.get(programId); + //key is UUID, want to filter by DBID + List studies = new ArrayList<>(); + if (cache != null) { + studies = cache.values().stream().filter(x -> studyDbIds.contains(x.getStudyDbId())).collect(Collectors.toList()); + } + return studies; + } + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java new file mode 100644 index 000000000..57c079d91 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java @@ -0,0 +1,119 @@ +package org.breedinginsight.brapi.v2.model.request.query; + +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import org.breedinginsight.api.model.v1.request.query.FilterRequest; +import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Introspected +public class StudyQuery extends BrapiQuery { + private Boolean active; + private List commonCropNames; + private List externalReferenceIds; + private List externalReferenceSources; + private List germplasmDbIds; + private List germplasmNames; + private List locationDbIds; + private List locationNames; + private List observationVariableDbIds; + private List observationVariableNames; + private List observationVariablePUIs; + private List programDbIds; + private List programNames; + private List seasonDbIds; + private List studyCodes; + private List studyDbIds; + private List studyNames; + private List studyPUIs; + private List studyTypes; + private List trialDbIds; + private List trialNames; + + + // TODO: need a more dynamic way to build FilterField, FilterRequest only handles a subset of filtering. + public SearchRequest constructSearchRequest() { + List filters = new ArrayList<>(); +// if (getActive()) { +// filters.add(constructFilterRequest("active", getActive())); +// } +// if (!StringUtils.isBlank(getCommonCropNames())) { +// filters.add(constructFilterRequest("commonCropName", getCommonCropNames())); +// } +// if (!StringUtils.isBlank(getExternalReferenceIds())) { +// filters.add(constructFilterRequest("externalReferenceId", getExternalReferenceIds())); +// } +// if (!StringUtils.isBlank(getExternalReferenceSources())) { +// filters.add(constructFilterRequest("externalReferenceSource", getExternalReferenceSources())); +// } +// if (!StringUtils.isBlank(getGermplasmDbIds())) { +// filters.add(constructFilterRequest("germplasmDbId", getGermplasmDbIds())); +// } +// if (!StringUtils.isBlank(getGermplasmNames())) { +// filters.add(constructFilterRequest("germplasmName", getGermplasmNames())); +// } +// if (!StringUtils.isBlank(getLocationDbIds())) { +// filters.add(constructFilterRequest("locationDbId", getLocationDbIds())); +// } +// if (!StringUtils.isBlank(getLocationNames())) { +// filters.add(constructFilterRequest("locationName", getLocationNames())); +// } +// if (!StringUtils.isBlank(getObservationVariableDbIds())) { +// filters.add(constructFilterRequest("observationVariableDbId", getObservationVariableDbIds())); +// } +// if (!StringUtils.isBlank(getObservationVariableNames())) { +// filters.add(constructFilterRequest("observationVariableName", getObservationVariableNames())); +// } +// if (!StringUtils.isBlank(getObservationVariablePUIs())) { +// filters.add(constructFilterRequest("observationVariablePUI", getObservationVariablePUIs())); +// } +// if (!StringUtils.isBlank(getPage())) { +// filters.add(constructFilterRequest("page", getPage())); +// } +// if (!StringUtils.isBlank(getPageSize())) { +// filters.add(constructFilterRequest("pageSize", getPageSize())); +// } +// if (!StringUtils.isBlank(getProgramDbIds())) { +// filters.add(constructFilterRequest("programDbId", getProgramDbIds())); +// } +// if (!StringUtils.isBlank(getProgramNames())) { +// filters.add(constructFilterRequest("programName", getProgramNames())); +// } +// if (!StringUtils.isBlank(getSeasonDbIds())) { +// filters.add(constructFilterRequest("seasonDbId", getSeasonDbIds())); +// } +// if (!StringUtils.isBlank(getSortBy())) { +// filters.add(constructFilterRequest("sortBy", getSortBy())); +// } +// if (!StringUtils.isBlank(getSortOrder())) { +// filters.add(constructFilterRequest("sortOrder", getSortOrder())); +// } +// if (!StringUtils.isBlank(getStudyCodes())) { +// filters.add(constructFilterRequest("studyCode", getStudyCodes())); +// } +// if (!StringUtils.isBlank(getStudyDbIds())) { +// filters.add(constructFilterRequest("studyDbId", getStudyDbIds())); +// } +// if (!StringUtils.isBlank(getStudyNames())) { +// filters.add(constructFilterRequest("studyName", getStudyNames())); +// } +// if (!StringUtils.isBlank(getStudyPUIs())) { +// filters.add(constructFilterRequest("studyPUI", getStudyPUIs())); +// } +// if (!StringUtils.isBlank(getStudyTypes())) { +// filters.add(constructFilterRequest("studyType", getStudyTypes())); +// } +// if (!StringUtils.isBlank(getTrialDbIds())) { +// filters.add(constructFilterRequest("trialDbId", getTrialDbIds())); +// } +// if (!StringUtils.isBlank(getTrialNames())) { +// filters.add(constructFilterRequest("trialName", getTrialNames())); +// } + + return new SearchRequest(filters); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java new file mode 100644 index 000000000..08d94d7b8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java @@ -0,0 +1,59 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.services.exceptions.DoesNotExistException; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; +import java.util.UUID; + +@Slf4j +@Singleton +public class BrAPIStudyService { + + private final BrAPIStudyDAO studyDAO; + + @Inject + public BrAPIStudyService(BrAPIStudyDAO studyDAO) { + this.studyDAO = studyDAO; + } + + public List getStudies(UUID programId) throws ApiException { + try { + return studyDAO.getStudies(programId); + } catch (ApiException e) { + throw new InternalServerException(e.getMessage(), e); + } + } + + public BrAPIStudy getStudyByUUID(UUID programId, String studyId) throws DoesNotExistException { + try { + return studyDAO.getStudyByUUID(studyId, programId); + } catch (ApiException e) { + throw new InternalServerException(e.getMessage(), e); + } + } + +} diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java new file mode 100644 index 000000000..41b1cd1c8 --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java @@ -0,0 +1,54 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.utilities.response.mappers; + +import lombok.Getter; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.api.v1.controller.metadata.SortOrder; + +import javax.inject.Singleton; +import java.util.Map; +import java.util.function.Function; + + +@Getter +@Singleton +public class StudyQueryMapper extends AbstractQueryMapper { + + private final String defaultSortField = "studyName"; + private final SortOrder defaultSortOrder = SortOrder.ASC; + private Map> fields; + + public StudyQueryMapper() { + fields = Map.ofEntries( + Map.entry("studyName", BrAPIStudy::getStudyName), + Map.entry("trialDbId", BrAPIStudy::getTrialDbId) + ); + } + + @Override + public boolean exists(String fieldName) { + return getFields().containsKey(fieldName); + } + + @Override + public Function getField(String fieldName) throws NullPointerException { + if (fields.containsKey(fieldName)) return fields.get(fieldName); + else throw new NullPointerException(); + } +} From 7d0ea06d318b9d79ebd53bc7959f0a29a62e3706 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Sun, 16 Jul 2023 16:14:20 -0400 Subject: [PATCH 13/25] [BI-1720] - TODO --- .../java/org/breedinginsight/brapi/v2/GermplasmController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index d91f4ed57..e27715e2e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -72,6 +72,7 @@ public GermplasmController(BrAPIGermplasmService germplasmService, GermplasmQuer this.brAPIEndpointProvider = brAPIEndpointProvider; } + // TODO: remove or update. I don't think this is used, and it doesn't properly support BrAPI request body. @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/search/germplasm{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) From 11172c82762072847cd9e11cdca7403588b3d65c Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Sun, 16 Jul 2023 16:15:01 -0400 Subject: [PATCH 14/25] [BI-1720] - DRY --- .../breedinginsight/brapps/importer/daos/BrAPITrialDAO.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index 0a9180936..ea37e22b7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -38,15 +38,11 @@ @Singleton public class BrAPITrialDAO { - @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; - private final ProgramDAO programDAO; private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final ProgramService programService; private final BrAPIEndpointProvider brAPIEndpointProvider; - private final String referenceSource; @Inject @@ -138,7 +134,7 @@ private List processExperimentsForDisplay(List trials, S public Optional getTrialById(UUID programId, UUID trialDbId) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - String refSoure = Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS); + String refSoure = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS); List trials = getTrialsByExRef(refSoure, trialDbId.toString(), program); return Utilities.getSingleOptional(trials); From 84f7f5481971bf96887f6fde1897b446da9c679a Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:10:54 -0400 Subject: [PATCH 15/25] [BI-1720] - study GET endpoint filtering working --- .../brapi/v2/StudyController.java | 27 +--- .../v2/model/request/query/StudyQuery.java | 132 +++++------------- .../brapi/v2/services/BrAPITrialService.java | 26 ++-- .../breedinginsight/utilities/Utilities.java | 36 ++++- .../response/mappers/StudyQueryMapper.java | 10 +- 5 files changed, 96 insertions(+), 135 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java index 69e5697a5..26758084c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java @@ -17,7 +17,6 @@ import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.model.v1.validators.QueryValid; import org.breedinginsight.brapi.v1.controller.BrapiVersion; -import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; import org.breedinginsight.brapi.v2.services.BrAPIStudyService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -44,26 +43,6 @@ public StudyController(BrAPIStudyService studyService, StudyQueryMapper studyQue this.studyQueryMapper = studyQueryMapper; } - @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/search/studies{?queryParams*}") - @Produces(MediaType.APPLICATION_JSON) - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse>>> searchStudy( - @PathVariable("programId") UUID programId, - @QueryValue @QueryValid(using = StudyQueryMapper.class) @Valid BrapiQuery queryParams, - @Body @Valid StudyQuery body) { - try { - log.debug("fetching studies for program: " + programId); - List studies = studyService.getStudies(programId); - queryParams.setSortField(studyQueryMapper.getDefaultSortField()); - queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder()); - // Filter the response. - return ResponseUtils.getBrapiQueryResponse(studies, studyQueryMapper, queryParams, body.constructSearchRequest()); - } catch (ApiException e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); - } - } - @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/studies{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) @@ -73,9 +52,11 @@ public HttpResponse>>> getStudy( try { log.debug("fetching studies for program: " + programId); - List study = studyService.getStudies(programId); + List studies = studyService.getStudies(programId); + queryParams.setSortField(studyQueryMapper.getDefaultSortField()); + queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder()); SearchRequest searchRequest = queryParams.constructSearchRequest(); - return ResponseUtils.getBrapiQueryResponse(study, studyQueryMapper, queryParams, searchRequest); + return ResponseUtils.getBrapiQueryResponse(studies, studyQueryMapper, queryParams, searchRequest); } catch (ApiException e) { log.info(e.getMessage(), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java index 57c079d91..c3115f05e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java @@ -5,6 +5,7 @@ import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; +import org.jooq.tools.StringUtils; import java.util.ArrayList; import java.util.List; @@ -12,108 +13,41 @@ @Getter @Introspected public class StudyQuery extends BrapiQuery { - private Boolean active; - private List commonCropNames; - private List externalReferenceIds; - private List externalReferenceSources; - private List germplasmDbIds; - private List germplasmNames; - private List locationDbIds; - private List locationNames; - private List observationVariableDbIds; - private List observationVariableNames; - private List observationVariablePUIs; - private List programDbIds; - private List programNames; - private List seasonDbIds; - private List studyCodes; - private List studyDbIds; - private List studyNames; - private List studyPUIs; - private List studyTypes; - private List trialDbIds; - private List trialNames; + private String studyType; + private String locationDbId; + private String studyCode; + private String studyPUI; + private String commonCropName; + private String trialDbId; + private String studyDbId; + private String studyName; - - // TODO: need a more dynamic way to build FilterField, FilterRequest only handles a subset of filtering. public SearchRequest constructSearchRequest() { List filters = new ArrayList<>(); -// if (getActive()) { -// filters.add(constructFilterRequest("active", getActive())); -// } -// if (!StringUtils.isBlank(getCommonCropNames())) { -// filters.add(constructFilterRequest("commonCropName", getCommonCropNames())); -// } -// if (!StringUtils.isBlank(getExternalReferenceIds())) { -// filters.add(constructFilterRequest("externalReferenceId", getExternalReferenceIds())); -// } -// if (!StringUtils.isBlank(getExternalReferenceSources())) { -// filters.add(constructFilterRequest("externalReferenceSource", getExternalReferenceSources())); -// } -// if (!StringUtils.isBlank(getGermplasmDbIds())) { -// filters.add(constructFilterRequest("germplasmDbId", getGermplasmDbIds())); -// } -// if (!StringUtils.isBlank(getGermplasmNames())) { -// filters.add(constructFilterRequest("germplasmName", getGermplasmNames())); -// } -// if (!StringUtils.isBlank(getLocationDbIds())) { -// filters.add(constructFilterRequest("locationDbId", getLocationDbIds())); -// } -// if (!StringUtils.isBlank(getLocationNames())) { -// filters.add(constructFilterRequest("locationName", getLocationNames())); -// } -// if (!StringUtils.isBlank(getObservationVariableDbIds())) { -// filters.add(constructFilterRequest("observationVariableDbId", getObservationVariableDbIds())); -// } -// if (!StringUtils.isBlank(getObservationVariableNames())) { -// filters.add(constructFilterRequest("observationVariableName", getObservationVariableNames())); -// } -// if (!StringUtils.isBlank(getObservationVariablePUIs())) { -// filters.add(constructFilterRequest("observationVariablePUI", getObservationVariablePUIs())); -// } -// if (!StringUtils.isBlank(getPage())) { -// filters.add(constructFilterRequest("page", getPage())); -// } -// if (!StringUtils.isBlank(getPageSize())) { -// filters.add(constructFilterRequest("pageSize", getPageSize())); -// } -// if (!StringUtils.isBlank(getProgramDbIds())) { -// filters.add(constructFilterRequest("programDbId", getProgramDbIds())); -// } -// if (!StringUtils.isBlank(getProgramNames())) { -// filters.add(constructFilterRequest("programName", getProgramNames())); -// } -// if (!StringUtils.isBlank(getSeasonDbIds())) { -// filters.add(constructFilterRequest("seasonDbId", getSeasonDbIds())); -// } -// if (!StringUtils.isBlank(getSortBy())) { -// filters.add(constructFilterRequest("sortBy", getSortBy())); -// } -// if (!StringUtils.isBlank(getSortOrder())) { -// filters.add(constructFilterRequest("sortOrder", getSortOrder())); -// } -// if (!StringUtils.isBlank(getStudyCodes())) { -// filters.add(constructFilterRequest("studyCode", getStudyCodes())); -// } -// if (!StringUtils.isBlank(getStudyDbIds())) { -// filters.add(constructFilterRequest("studyDbId", getStudyDbIds())); -// } -// if (!StringUtils.isBlank(getStudyNames())) { -// filters.add(constructFilterRequest("studyName", getStudyNames())); -// } -// if (!StringUtils.isBlank(getStudyPUIs())) { -// filters.add(constructFilterRequest("studyPUI", getStudyPUIs())); -// } -// if (!StringUtils.isBlank(getStudyTypes())) { -// filters.add(constructFilterRequest("studyType", getStudyTypes())); -// } -// if (!StringUtils.isBlank(getTrialDbIds())) { -// filters.add(constructFilterRequest("trialDbId", getTrialDbIds())); -// } -// if (!StringUtils.isBlank(getTrialNames())) { -// filters.add(constructFilterRequest("trialName", getTrialNames())); -// } - + if (!StringUtils.isBlank(getStudyType())) { + filters.add(constructFilterRequest("studyType", getStudyType())); + } + if (!StringUtils.isBlank(getLocationDbId())) { + filters.add(constructFilterRequest("locationDbId", getLocationDbId())); + } + if (!StringUtils.isBlank(getStudyCode())) { + filters.add(constructFilterRequest("studyCode", getStudyCode())); + } + if (!StringUtils.isBlank(getStudyPUI())) { + filters.add(constructFilterRequest("studyPUI", getStudyPUI())); + } + if (!StringUtils.isBlank(getCommonCropName())) { + filters.add(constructFilterRequest("commonCropName", getCommonCropName())); + } + if (!StringUtils.isBlank(getTrialDbId())) { + filters.add(constructFilterRequest("trialDbId", getTrialDbId())); + } + if (!StringUtils.isBlank(getStudyDbId())) { + filters.add(constructFilterRequest("studyDbId", getStudyDbId())); + } + if (!StringUtils.isBlank(getStudyName())) { + filters.add(constructFilterRequest("studyName", getStudyName())); + } return new SearchRequest(filters); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 0126926a4..c4c5576dd 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,4 +1,5 @@ package org.breedinginsight.brapi.v2.services; + import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -28,6 +29,7 @@ import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.utilities.Utilities; + import javax.inject.Inject; import javax.inject.Singleton; import java.io.IOException; @@ -226,7 +228,7 @@ public DownloadFile exportObservations( else { // Zip, as there are multiple files. StreamedFile zipFile = zipFiles(files); - downloadFile = new DownloadFile(makeZipFileName(experiment), zipFile); + downloadFile = new DownloadFile(makeZipFileName(experiment, program), zipFile); } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); @@ -465,22 +467,26 @@ private void addObsVarColumns( } } private String makeFileName(BrAPITrial experiment, Program program, String envName) { - // _Observation Dataset [-]__ - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + // _Observation Dataset__ + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - return String.format("%s_Observation Dataset [%s-%s]_%s_%s", + String unsafeName = String.format("%s_Observation Dataset_%s_%s", Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), - program.getKey(), - experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(), - envName, + Utilities.removeProgramKeyAndUnknownAdditionalData(envName, program.getKey()), timestamp); + // Make file name safe for all platforms. + return Utilities.makePortableFilename(unsafeName); } - private String makeZipFileName(BrAPITrial experiment) { + private String makeZipFileName(BrAPITrial experiment, Program program) { // .zip - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - return String.format("%s_%s.zip", experiment.getTrialName(), timestamp); + String unsafeName = String.format("%s_%s.zip", + Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), + timestamp); + // Make file name safe for all platforms. + return Utilities.makePortableFilename(unsafeName); } private List filterDatasetByEnvironment( diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 52e85ad2d..9ce3a1b9e 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import java.util.List; @@ -152,4 +151,39 @@ public static Optional getSingleOptional(List items) { return Optional.empty(); } } + + /** + * For a possibly unsafe file name, return a new String that is safe across platforms. + * @param name a possibly unsafe file name + * @return a portable file name + */ + public static String makePortableFilename(String name) { + StringBuilder sb = new StringBuilder(); + char c; + char last_appended = '_'; + int i = 0; + while (i < name.length()) { + c = name.charAt(i); + if (isSafeChar(c)) { + sb.append(c); + last_appended = c; + } + else { + // Replace illegal chars with '_', but prevent repeat underscores. + if (last_appended != '_') { + sb.append('_'); + last_appended = '_'; + } + } + ++i; + } + + return sb.toString(); + } + + private static boolean isSafeChar(char c) { + // Check if c is in the portable filename character set. + // See https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html#tag_03_282 + return Character.isLetterOrDigit(c) || c == '-' || c == '_' || c == '.'; + } } diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java index 41b1cd1c8..96311c00b 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java @@ -36,8 +36,14 @@ public class StudyQueryMapper extends AbstractQueryMapper { public StudyQueryMapper() { fields = Map.ofEntries( - Map.entry("studyName", BrAPIStudy::getStudyName), - Map.entry("trialDbId", BrAPIStudy::getTrialDbId) + Map.entry("studyType", BrAPIStudy::getStudyType), + Map.entry("locationDbId", BrAPIStudy::getLocationDbId), + Map.entry("studyCode", BrAPIStudy::getStudyCode), + Map.entry("studyPUI", BrAPIStudy::getStudyPUI), + Map.entry("commonCropName", BrAPIStudy::getCommonCropName), + Map.entry("trialDbId", BrAPIStudy::getTrialDbId), + Map.entry("studyDbId", BrAPIStudy::getStudyDbId), + Map.entry("studyName", BrAPIStudy::getStudyName) ); } From 6d3040e6249f6bd005fd656388dfcccccce686fa Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:39:29 -0400 Subject: [PATCH 16/25] [BI-1720] - removed unused controller method --- .../brapi/v2/StudyController.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java index 26758084c..27a9033aa 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java @@ -65,24 +65,4 @@ public HttpResponse>>> getStudy( return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); } } - - @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/studies/{studyId}") - @Produces(MediaType.APPLICATION_JSON) - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse> getSingleStudy( - @PathVariable("programId") UUID programId, - @PathVariable("studyId") String studyId) { - try { - log.debug("fetching study id:" + studyId +" for program: " + programId); - Response response = new Response(studyService.getStudyByUUID(programId, studyId)); - return HttpResponse.ok(response); - } catch (InternalServerException e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); - } catch (DoesNotExistException e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.NOT_FOUND, "Study not found"); - } - } - } From 6e0b36f81f6f9fa3199fc8de6cf63c07b6fa4535 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:42:45 -0400 Subject: [PATCH 17/25] [BI-1720] - updated TODO --- .../java/org/breedinginsight/brapi/v2/GermplasmController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index e27715e2e..5a022d88d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -72,7 +72,7 @@ public GermplasmController(BrAPIGermplasmService germplasmService, GermplasmQuer this.brAPIEndpointProvider = brAPIEndpointProvider; } - // TODO: remove or update. I don't think this is used, and it doesn't properly support BrAPI request body. + // TODO: expand to fully support BrAPI request body. @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/search/germplasm{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) From be733240b7a01920858fcce9446d3cc10257e2b3 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:54:27 -0400 Subject: [PATCH 18/25] [BI-1720] - renamed method --- src/main/java/org/breedinginsight/brapi/v2/StudyController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java index 27a9033aa..5edfb7939 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java @@ -46,7 +46,7 @@ public StudyController(BrAPIStudyService studyService, StudyQueryMapper studyQue @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/studies{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse>>> getStudy( + public HttpResponse>>> getStudies( @PathVariable("programId") UUID programId, @QueryValue @QueryValid(using = StudyQueryMapper.class) @Valid StudyQuery queryParams) { try { From 0342789410d01a1e1869ec2dfa9f5cb831f16ed9 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:00:34 -0400 Subject: [PATCH 19/25] [BI-1720] - removed unused methods --- .../brapi/v2/dao/BrAPIStudyDAO.java | 32 ------------------- .../brapi/v2/services/BrAPIStudyService.java | 9 ------ 2 files changed, 41 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java index 53ec5d5b0..f171f7652 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -163,36 +163,4 @@ private Map processStudyForDisplay(List programSt return programStudyMap; } - public BrAPIStudy getStudyByUUID(String studyId, UUID programId) throws ApiException, DoesNotExistException { - Map cache = programStudyCache.get(programId); - BrAPIStudy study = null; - if (cache != null) { - study = cache.get(studyId); - } - if (study == null) { - throw new DoesNotExistException("UUID for this study does not exist"); - } - return study; - } - - public Optional getStudyByDBID(String studyDbId, UUID programId) throws ApiException { - Map cache = programStudyCache.get(programId); - //key is UUID, want to filter by DBID - BrAPIStudy study = null; - if (cache != null) { - study = cache.values().stream().filter(x -> x.getStudyDbId().equals(studyDbId)).collect(Collectors.toList()).get(0); - } - return Optional.ofNullable(study); - } - - public List getStudiesByDBID(Collection studyDbIds, UUID programId) throws ApiException { - Map cache = programStudyCache.get(programId); - //key is UUID, want to filter by DBID - List studies = new ArrayList<>(); - if (cache != null) { - studies = cache.values().stream().filter(x -> studyDbIds.contains(x.getStudyDbId())).collect(Collectors.toList()); - } - return studies; - } - } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java index 08d94d7b8..6e1e6281f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java @@ -22,7 +22,6 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; -import org.breedinginsight.services.exceptions.DoesNotExistException; import javax.inject.Inject; import javax.inject.Singleton; @@ -48,12 +47,4 @@ public List getStudies(UUID programId) throws ApiException { } } - public BrAPIStudy getStudyByUUID(UUID programId, String studyId) throws DoesNotExistException { - try { - return studyDAO.getStudyByUUID(studyId, programId); - } catch (ApiException e) { - throw new InternalServerException(e.getMessage(), e); - } - } - } From 47af603e1334166737f1df317cdefa1bdb1f1c50 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:00:39 -0400 Subject: [PATCH 20/25] [BI-1720] - clean up --- .../brapi/v2/StudyController.java | 19 +++++++++++++++++-- .../brapi/v2/dao/BrAPIStudyDAO.java | 5 +---- .../response/mappers/StudyQueryMapper.java | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java index 5edfb7939..29aeb7dd3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/StudyController.java @@ -1,10 +1,26 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapi.v2; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; -import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; @@ -19,7 +35,6 @@ import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; import org.breedinginsight.brapi.v2.services.BrAPIStudyService; -import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.StudyQueryMapper; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java index f171f7652..3346b9747 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -34,7 +34,6 @@ import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; -import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; @@ -127,10 +126,8 @@ private Map fetchProgramStudy(UUID programId) throws ApiExce private Map processStudyForDisplay(List programStudy, String programKey) { // Process the study Map programStudyMap = new HashMap<>(); - log.trace("processing germ for display: " + programStudy); - Map programStudyByFullName = new HashMap<>(); + log.trace("processing study for display: " + programStudy); for (BrAPIStudy study: programStudy) { - programStudyByFullName.put(study.getStudyName(), study); // Remove program key from studyName, trialName and locationName. if (study.getStudyName() != null) { // Study name is appended with program key and experiment sequence number, need to remove. diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java index 96311c00b..a6007c2d6 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java @@ -32,7 +32,7 @@ public class StudyQueryMapper extends AbstractQueryMapper { private final String defaultSortField = "studyName"; private final SortOrder defaultSortOrder = SortOrder.ASC; - private Map> fields; + private final Map> fields; public StudyQueryMapper() { fields = Map.ofEntries( From fdb4c4d8429c050098ca47a5fad76fc8c675d990 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jul 2023 15:11:54 -0400 Subject: [PATCH 21/25] [BI-1720] - reverted ListQuery change changed when externalReferenceSource was made a different type, wasn't changed back when it was made a String again --- .../brapi/v2/model/request/query/ListQuery.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ListQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ListQuery.java index c1b9c6f33..97646c61d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ListQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ListQuery.java @@ -5,7 +5,6 @@ import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.jooq.tools.StringUtils; import java.util.ArrayList; @@ -39,7 +38,7 @@ public SearchRequest constructSearchRequest() { if (!StringUtils.isBlank(getSize())) { filters.add(constructFilterRequest("size", getSize())); } - if (getExternalReferenceSource() != null) { + if (!StringUtils.isBlank(getExternalReferenceSource())) { filters.add(constructFilterRequest("externalReferenceSource", getExternalReferenceSource())); } if (!StringUtils.isBlank(getExternalReferenceId())) { From 7c0b3f7ad44d31f146ae9ad780c5383382127f28 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jul 2023 15:17:29 -0400 Subject: [PATCH 22/25] [BI-1720] - added exRef source and ID to study query --- .../brapi/v2/model/request/query/StudyQuery.java | 8 ++++++++ .../response/mappers/StudyQueryMapper.java | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java index c3115f05e..5c3d6bfeb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java @@ -21,6 +21,8 @@ public class StudyQuery extends BrapiQuery { private String trialDbId; private String studyDbId; private String studyName; + private String externalReferenceSource; + private String externalReferenceId; public SearchRequest constructSearchRequest() { List filters = new ArrayList<>(); @@ -48,6 +50,12 @@ public SearchRequest constructSearchRequest() { if (!StringUtils.isBlank(getStudyName())) { filters.add(constructFilterRequest("studyName", getStudyName())); } + if (!StringUtils.isBlank(getExternalReferenceSource())) { + filters.add(constructFilterRequest("externalReferenceSource", getExternalReferenceSource())); + } + if (!StringUtils.isBlank(getExternalReferenceId())) { + filters.add(constructFilterRequest("externalReferenceId", getExternalReferenceId())); + } return new SearchRequest(filters); } } diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java index a6007c2d6..f15aa620f 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java @@ -18,17 +18,19 @@ package org.breedinginsight.utilities.response.mappers; import lombok.Getter; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.api.v1.controller.metadata.SortOrder; import javax.inject.Singleton; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; @Getter @Singleton -public class StudyQueryMapper extends AbstractQueryMapper { +public class StudyQueryMapper extends AbstractQueryMapper { private final String defaultSortField = "studyName"; private final SortOrder defaultSortOrder = SortOrder.ASC; @@ -43,7 +45,17 @@ public StudyQueryMapper() { Map.entry("commonCropName", BrAPIStudy::getCommonCropName), Map.entry("trialDbId", BrAPIStudy::getTrialDbId), Map.entry("studyDbId", BrAPIStudy::getStudyDbId), - Map.entry("studyName", BrAPIStudy::getStudyName) + Map.entry("studyName", BrAPIStudy::getStudyName), + Map.entry("externalReferenceSource", (study) -> study + .getExternalReferences() + .stream() + .map(BrAPIExternalReference::getReferenceSource) + .collect(Collectors.toList())), + Map.entry("externalReferenceId", (study) -> study + .getExternalReferences() + .stream() + .map(BrAPIExternalReference::getReferenceID) + .collect(Collectors.toList())) ); } From 024f4e6d6d497fc8a91111a8c41a3520094b22b8 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jul 2023 16:59:17 -0400 Subject: [PATCH 23/25] [BI-1720] - merged redundant BrAPIStudyDAO --- .../brapi/v2/dao/BrAPIStudyDAO.java | 163 ------------------ .../brapi/v2/services/BrAPIStudyService.java | 2 +- .../brapps/importer/daos/BrAPIStudyDAO.java | 113 +++++++++++- 3 files changed, 111 insertions(+), 167 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java deleted file mode 100644 index 3346b9747..000000000 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapi.v2.dao; - -import com.google.gson.JsonObject; -import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.modules.core.StudiesApi; -import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; -import org.breedinginsight.daos.cache.ProgramCacheProvider; -import org.breedinginsight.model.Program; -import org.breedinginsight.services.brapi.BrAPIEndpointProvider; -import org.breedinginsight.utilities.BrAPIDAOUtil; -import org.breedinginsight.utilities.Utilities; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -@Singleton -@Context -public class BrAPIStudyDAO { - - private final ProgramDAO programDAO; - private final BrAPIDAOUtil brAPIDAOUtil; - - @Property(name = "brapi.server.reference-source") - private String referenceSource; - - @Property(name = "micronaut.bi.api.run-scheduled-tasks") - private boolean runScheduledTasks; - - private final ProgramCache programStudyCache; - - private final BrAPIEndpointProvider brAPIEndpointProvider; - - @Inject - public BrAPIStudyDAO(ProgramDAO programDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramCacheProvider programCacheProvider, BrAPIEndpointProvider brAPIEndpointProvider) { - this.programDAO = programDAO; - this.brAPIDAOUtil = brAPIDAOUtil; - this.programStudyCache = programCacheProvider.getProgramCache(this::fetchProgramStudy, BrAPIStudy.class); - this.brAPIEndpointProvider = brAPIEndpointProvider; - } - - @Scheduled(initialDelay = "2s") - public void setup() { - if(!runScheduledTasks) { - return; - } - // Populate study cache for all programs on startup - log.debug("populating study cache"); - List programs = programDAO.getActive(); - if(programs != null) { - programStudyCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } - } - - /** - * Fetch the study for this program, and process it to remove storage specific values - * @param programId - * @return this program's study - * @throws ApiException - */ - public List getStudies(UUID programId) throws ApiException { - return new ArrayList<>(programStudyCache.get(programId).values()); - } - - /** - * Fetch formatted study for this program - * @param programId - * @return Map - * @throws ApiException - */ - private Map fetchProgramStudy(UUID programId) throws ApiException { - StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); - // Get the program key - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given key"); - } - Program program = programs.get(0); - - // Set query params and make call - BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); - studySearch.externalReferenceIDs(List.of(programId.toString())); - studySearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processStudyForDisplay(brAPIDAOUtil.search( - api::searchStudiesPost, - api::searchStudiesSearchResultsDbIdGet, - studySearch - ), program.getKey()); - } - - /** - * Process study into a format for display - * @param programStudy - * @return Map - * @throws ApiException - */ - private Map processStudyForDisplay(List programStudy, String programKey) { - // Process the study - Map programStudyMap = new HashMap<>(); - log.trace("processing study for display: " + programStudy); - for (BrAPIStudy study: programStudy) { - // Remove program key from studyName, trialName and locationName. - if (study.getStudyName() != null) { - // Study name is appended with program key and experiment sequence number, need to remove. - study.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey)); - } - if (study.getTrialName() != null) { - study.setTrialName(Utilities.removeProgramKey(study.getTrialName(), programKey)); - } - if (study.getLocationName() != null) { - study.setLocationName(Utilities.removeProgramKey(study.getLocationName(), programKey)); - } - } - - String refSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); - // Add to map. - for (BrAPIStudy study: programStudy) { - JsonObject additionalInfo = study.getAdditionalInfo(); - if(additionalInfo == null) { - additionalInfo = new JsonObject(); - study.setAdditionalInfo(additionalInfo); - } - - BrAPIExternalReference extRef = study.getExternalReferences().stream() - .filter(reference -> reference.getReferenceSource().equals(refSource)) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - String studyId = extRef.getReferenceID(); - programStudyMap.put(studyId, study); - } - - return programStudyMap; - } - -} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java index 6e1e6281f..f2069e5bb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; import javax.inject.Inject; import javax.inject.Singleton; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index 456c7adf5..15bf3acfb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -16,14 +16,21 @@ */ package org.breedinginsight.brapps.importer.daos; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.scheduling.annotation.Scheduled; +import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.core.StudiesApi; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.cache.ProgramCache; +import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; @@ -33,29 +40,87 @@ import javax.inject.Singleton; import javax.validation.constraints.NotNull; import java.util.*; +import java.util.stream.Collectors; +@Slf4j @Singleton public class BrAPIStudyDAO { @Property(name = "brapi.server.reference-source") - private String BRAPI_REFERENCE_SOURCE; + private String referenceSource; + @Property(name = "micronaut.bi.api.run-scheduled-tasks") + private boolean runScheduledTasks; private ProgramDAO programDAO; private ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final ProgramCache programStudyCache; + @Inject - public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, ProgramCacheProvider programCacheProvider) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; + this.programStudyCache = programCacheProvider.getProgramCache(this::fetchProgramStudy, BrAPIStudy.class); + } + + @Scheduled(initialDelay = "2s") + public void setup() { + if(!runScheduledTasks) { + return; + } + // Populate study cache for all programs on startup + log.debug("populating study cache"); + List programs = programDAO.getActive(); + if(programs != null) { + programStudyCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + } + } + + + /** + * Fetch formatted study for this program + * @param programId + * @return Map - Key = string representing study UUID, value = formatted BrAPIStudy + * @throws ApiException + */ + private Map fetchProgramStudy(UUID programId) throws ApiException { + StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); + // Get the program key + List programs = programDAO.get(programId); + if (programs.size() != 1) { + throw new InternalServerException("Program was not found for given key"); + } + Program program = programs.get(0); + + // Set query params and make call + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); + studySearch.externalReferenceIDs(List.of(programId.toString())); + studySearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); + return processStudyForDisplay(brAPIDAOUtil.search( + api::searchStudiesPost, + api::searchStudiesSearchResultsDbIdGet, + studySearch + ), program.getKey()); + } + + /** + * Fetch the study for this program, and process it to remove storage specific values + * @param programId + * @return this program's study + * @throws ApiException + */ + public List getStudies(UUID programId) throws ApiException { + return new ArrayList<>(programStudyCache.get(programId).values()); } public Optional getStudyByName(String studyName, Program program) throws ApiException { List studies = getStudiesByName(List.of(studyName), program); return Utilities.getSingleOptional(studies); } + public List getStudiesByName(List studyNames, Program program) throws ApiException { if(studyNames.isEmpty()) { return Collections.emptyList(); @@ -76,7 +141,7 @@ public List getStudiesByExperimentID(@NotNull UUID experimentID, Pro BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); studySearch.addExternalReferenceIDsItem(experimentID.toString()); - studySearch.addExternalReferenceSourcesItem(Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS)); + studySearch.addExternalReferenceSourcesItem(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)); StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); return brAPIDAOUtil.search( api::searchStudiesPost, @@ -111,4 +176,46 @@ public Optional getStudyByDbId(String studyDbId, Program program) th return Utilities.getSingleOptional(studies); } + + + /** + * Process study into a format for display + * @param programStudy + * @return Map - Key = string representing study UUID, value = formatted BrAPIStudy + */ + private Map processStudyForDisplay(List programStudy, String programKey) { + Map programStudyMap = new HashMap<>(); + log.trace("processing study for display: " + programStudy); + for (BrAPIStudy study: programStudy) { + // Remove program key from studyName, trialName and locationName. + if (study.getStudyName() != null) { + // Study name is appended with experiment sequence number in addition to program key. + study.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey)); + } + if (study.getTrialName() != null) { + study.setTrialName(Utilities.removeProgramKey(study.getTrialName(), programKey)); + } + if (study.getLocationName() != null) { + study.setLocationName(Utilities.removeProgramKey(study.getLocationName(), programKey)); + } + } + + String refSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); + // Add to map. + for (BrAPIStudy study: programStudy) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if(additionalInfo == null) { + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + + BrAPIExternalReference extRef = study.getExternalReferences().stream() + .filter(reference -> reference.getReferenceSource().equals(refSource)) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + String studyId = extRef.getReferenceID(); + programStudyMap.put(studyId, study); + } + + return programStudyMap; + } } \ No newline at end of file From 88a4cd5f6b036ba2cb870dec1cf82879c8e2ab06 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:27:30 -0400 Subject: [PATCH 24/25] [BI-1720] - cache studies on create --- .../brapps/importer/daos/BrAPIStudyDAO.java | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index 15bf3acfb..10738e788 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -40,6 +40,7 @@ import javax.inject.Singleton; import javax.validation.constraints.NotNull; import java.util.*; +import java.util.concurrent.Callable; import java.util.stream.Collectors; @Slf4j @@ -152,7 +153,37 @@ public List getStudiesByExperimentID(@NotNull UUID experimentID, Pro public List createBrAPIStudies(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); - return brAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); + List createdStudies = new ArrayList<>(); + try { + if (!brAPIStudyList.isEmpty()) { + Callable> postCallback = () -> { + List postedStudies = brAPIDAOUtil + .post(brAPIStudyList, upload, api::studiesPost, importDAO::update); + return studyById(postedStudies); + }; + createdStudies.addAll(programStudyCache.post(programId, postCallback)); + } + + return createdStudies; + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + + /** + * @return Map - Key = BI external reference ID, Value = BrAPIStudy + * */ + private Map studyById(List studies) { + Map studyById = new HashMap<>(); + for (BrAPIStudy study: studies) { + BrAPIExternalReference xref = study + .getExternalReferences() + .stream() + .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES).equalsIgnoreCase(reference.getReferenceSource())) + .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); + studyById.put(xref.getReferenceID(), study); + } + return studyById; } public List getStudiesByStudyDbId(Collection studyDbIds, Program program) throws ApiException { From 6aa7cc835a3c5cc355f8b7c1225d1fa26d4a99b4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:32:35 -0400 Subject: [PATCH 25/25] [BI-1720] - made naming more consistent --- .../brapps/importer/daos/BrAPIStudyDAO.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index 10738e788..cae09d9fb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -159,7 +159,7 @@ public List createBrAPIStudies(List brAPIStudyList, UUID Callable> postCallback = () -> { List postedStudies = brAPIDAOUtil .post(brAPIStudyList, upload, api::studiesPost, importDAO::update); - return studyById(postedStudies); + return environmentById(postedStudies); }; createdStudies.addAll(programStudyCache.post(programId, postCallback)); } @@ -173,17 +173,17 @@ public List createBrAPIStudies(List brAPIStudyList, UUID /** * @return Map - Key = BI external reference ID, Value = BrAPIStudy * */ - private Map studyById(List studies) { - Map studyById = new HashMap<>(); - for (BrAPIStudy study: studies) { - BrAPIExternalReference xref = study + private Map environmentById(List studies) { + Map environmentById = new HashMap<>(); + for (BrAPIStudy environment: studies) { + BrAPIExternalReference xref = environment .getExternalReferences() .stream() .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES).equalsIgnoreCase(reference.getReferenceSource())) .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - studyById.put(xref.getReferenceID(), study); + environmentById.put(xref.getReferenceID(), environment); } - return studyById; + return environmentById; } public List getStudiesByStudyDbId(Collection studyDbIds, Program program) throws ApiException {