From bd0701e4ea7f7c088400b25d3ddc4fe0c40253ff Mon Sep 17 00:00:00 2001 From: timparsons Date: Tue, 25 Jul 2023 13:49:43 -0400 Subject: [PATCH] [BI-1864] Improving performance of experiment export --- .../request/query/ExperimentExportQuery.java | 2 + .../brapi/v2/services/BrAPITrialService.java | 37 +++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 5f60f1008..e0c4a5b19 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -2,6 +2,7 @@ import io.micronaut.core.annotation.Introspected; import lombok.Getter; +import lombok.ToString; import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; @@ -14,6 +15,7 @@ @Getter @Introspected +@ToString public class ExperimentExportQuery { private FileType fileExtension; private String dataset; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 4faa89dba..47622bfe4 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -32,6 +32,7 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; @Slf4j @@ -99,6 +100,8 @@ public DownloadFile exportObservations( Program program, UUID experimentId, ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException { + String logHash = UUID.randomUUID().toString(); + log.debug(logHash + ": exporting experiment: "+experimentId+", params: " + params); StreamedFile downloadFile; boolean isDataset = false; List dataset = new ArrayList<>(); @@ -110,6 +113,7 @@ public DownloadFile exportObservations( FileType fileType = params.getFileExtension(); // get requested environments for the experiment + log.debug(logHash + ": fetching environments for export"); List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); if (!requestedEnvIds.isEmpty()) { expStudies = expStudies @@ -120,6 +124,7 @@ public DownloadFile exportObservations( expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); // get the OUs for the requested environments + log.debug(logHash + ": fetching OUs for export"); List ous = new ArrayList<>(); Map ouByOUDbId = new HashMap<>(); try { @@ -129,7 +134,7 @@ public DownloadFile exportObservations( ous.addAll(studyOUs); } } catch (ApiException err) { - log.error("Error fetching observation units for a study by its DbId" + + log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } @@ -137,6 +142,7 @@ public DownloadFile exportObservations( List columns = ExperimentFileColumns.getOrderedColumns(); // add columns for requested dataset obsvars and timestamps + log.debug(logHash + ": fetching experiment for export"); BrAPITrial experiment = getExperiment(program, experimentId); if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { @@ -144,6 +150,7 @@ public DownloadFile exportObservations( .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); isDataset = true; + log.debug(logHash + ": fetching dataset observation variables for export"); obsVars = getDatasetObsVars(obsDatasetId, program); // make additional columns in the export for each obs variable and obs variable timestamp @@ -153,12 +160,19 @@ public DownloadFile exportObservations( // make export rows from any observations if (isDataset) { + log.debug(logHash + ": fetching observations for export"); dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); } if (!requestedEnvIds.isEmpty()) { + log.debug(logHash + ": filtering observations to only requested environments for export"); dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); } + log.debug(logHash + ": fetching program's germplasm for export"); + List programGermplasm = germplasmDAO.getGermplasmsByDBID(ouByOUDbId.values().stream().map(BrAPIObservationUnit::getGermplasmDbId).collect(Collectors.toList()), program.getId()); + Map programGermplasmByDbId = programGermplasm.stream().collect(Collectors.toMap(BrAPIGermplasm::getGermplasmDbId, Function.identity())); + + log.debug(logHash + ": populating rows for export"); // update rowByOUId addBrAPIObsToRecords( dataset, @@ -168,7 +182,8 @@ public DownloadFile exportObservations( studyByDbId, rowByOUId, params.isIncludeTimestamps(), - obsVars + obsVars, + programGermplasmByDbId ); // make export rows for OUs without observations @@ -176,11 +191,12 @@ public DownloadFile exportObservations( for (BrAPIObservationUnit ou: ous) { String ouId = getOUId(ou); if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId)); } } } + log.debug(logHash + ": writing data to file for export"); // write export data to requested file format List> exportRows = new ArrayList<>(rowByOUId.values()); if (fileType.equals(FileType.CSV)){ @@ -191,7 +207,10 @@ public DownloadFile exportObservations( String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); - return new DownloadFile(fileName, downloadFile); + DownloadFile retFile = new DownloadFile(fileName, downloadFile); + + log.debug(logHash + ": completed export of experiment: " + experimentId + ", params: " + params); + return retFile; } private void addBrAPIObsToRecords( @@ -202,7 +221,8 @@ private void addBrAPIObsToRecords( Map studyByDbId, Map> rowByOUId, boolean includeTimestamp, - List obsVars) throws ApiException, DoesNotExistException { + List obsVars, + Map programGermplasmByDbId) throws ApiException, DoesNotExistException { Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { @@ -220,7 +240,7 @@ private void addBrAPIObsToRecords( } else { // otherwise make a new row - Map row = createExportRow(experiment, program, ou, studyByDbId); + Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId); addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } @@ -291,7 +311,8 @@ private Map createExportRow( BrAPITrial experiment, Program program, BrAPIObservationUnit ou, - Map studyByDbId) throws ApiException, DoesNotExistException { + Map studyByDbId, + Map programGermplasmByDbId) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); // get OU id, germplasm, and study @@ -300,7 +321,7 @@ private Map createExportRow( String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) .orElseThrow(() -> new RuntimeException("observation unit id not found")); String ouId = ouXref.getReferenceID(); - BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(ou.getGermplasmDbId(), program.getId()) + BrAPIGermplasm germplasm = Optional.ofNullable(programGermplasmByDbId.get(ou.getGermplasmDbId())) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); BrAPIStudy study = studyByDbId.get(ou.getStudyDbId());