From ae4cdc4b1117504040427f359072dfed039e7cf4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 22 May 2024 18:40:34 -0400 Subject: [PATCH 01/28] [BI-2109] prepared BrAPITrialService for dataset work --- .../brapi/v2/services/BrAPITrialService.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..b0f821243 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -244,7 +244,8 @@ public DownloadFile exportObservations( List> rows = entry.getValue(); sortDefaultForExportRows(rows); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); - String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); + // TODO: remove hardcoded datasetName, use observation level. + String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); } @@ -265,7 +266,8 @@ public DownloadFile exportObservations( StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); + // TODO: remove hardcoded datasetName, use observation level. + String fileName = makeFileName(experiment, program, envFilenameFragment, "Observation Dataset") + fileType.getExtension(); downloadFile = new DownloadFile(fileName, streamedFile); } @@ -298,17 +300,17 @@ private StreamedFile zipFiles(List files) throws IOException { return new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); } - public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, Boolean stats) throws ApiException, DoesNotExistException { - log.debug("fetching dataset: " + datsetId + " for experiment: " + experimentId + ". including stats: " + stats); - log.debug("fetching observationUnits for dataset: " + datsetId); - List datasetOUs = ouDAO.getObservationUnitsForDataset(datsetId.toString(), program); - log.debug("fetching dataset variables dataset: " + datsetId); - List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); + public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId, Boolean stats) throws ApiException, DoesNotExistException { + log.debug("fetching dataset: " + datasetId + " for experiment: " + experimentId + ". including stats: " + stats); + log.debug("fetching observationUnits for dataset: " + datasetId); + List datasetOUs = ouDAO.getObservationUnitsForDataset(datasetId.toString(), program); + log.debug("fetching dataset variables dataset: " + datasetId); + List datasetObsVars = getDatasetObsVars(datasetId.toString(), program); List ouDbIds = datasetOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); List obsVarDbIds = datasetObsVars.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()); - log.debug("fetching observations for dataset: " + datsetId); + log.debug("fetching observations for dataset: " + datasetId); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); - log.debug("building dataset object for dataset: " + datsetId); + log.debug("building dataset object for dataset: " + datasetId); sortDefaultForObservationUnit(datasetOUs); Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { @@ -521,12 +523,14 @@ private void addObsVarColumns( } } } - private String makeFileName(BrAPITrial experiment, Program program, String envName) { - // _Observation Dataset__ + + private String makeFileName(BrAPITrial experiment, Program program, String envName, String datasetName) { + // ___ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - String unsafeName = String.format("%s_Observation Dataset_%s_%s", + String unsafeName = String.format("%s_%s_%s_%s", Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), + datasetName, Utilities.removeProgramKeyAndUnknownAdditionalData(envName, program.getKey()), timestamp); // Make file name safe for all platforms. From 7669e0ecd7a490ea44ed3048d0bf25757a2ebb98 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 23 May 2024 16:28:41 -0400 Subject: [PATCH 02/28] [BI-2109] added controller method and service method stub --- .../v1/request/SubEntityDatasetRequest.java | 38 +++++++++++++++++++ .../v1/controller/ExperimentController.java | 18 +++++++++ .../brapi/v2/services/BrAPITrialService.java | 16 ++++++++ 3 files changed, 72 insertions(+) create mode 100644 src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java b/src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java new file mode 100644 index 000000000..c45cfae31 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java @@ -0,0 +1,38 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.model.v1.request; + +import io.micronaut.core.annotation.Introspected; +import lombok.*; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +@Introspected +public class SubEntityDatasetRequest { + @NotBlank + private String name; + @NotNull + private int repeatedMeasures; +} diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 017197219..30a2c1fe4 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -11,6 +11,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.api.model.v1.request.SubEntityDatasetRequest; import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; @@ -19,6 +20,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; @@ -89,4 +91,20 @@ public HttpResponse> getDatasetData( } } + @Post("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/dataset") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse> getDatasetData( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId, + @Body @Valid SubEntityDatasetRequest datasetRequest) { + try { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); + Response response = new Response(experimentService.createSubEntityDataset(program, experimentId, datasetRequest)); + return HttpResponse.ok(response); + } catch (Exception e){ + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b0f821243..458e486a8 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -5,6 +5,7 @@ import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -13,6 +14,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.*; +import org.breedinginsight.api.model.v1.request.SubEntityDatasetRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; @@ -329,6 +331,20 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId return dataset; } + public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException { + log.debug("creating sub-entity dataset: \"" + request.getName() + "\" for experiment: \"" + experimentId + "\" with: \"" + request.getRepeatedMeasures() + "\" repeated measures."); + BrAPITrial experiment = getExperiment(program, experimentId); + String a = "b"; + List datasetOUs = ouDAO.getObservationUnitsForDataset(experiment.getAdditionalInfo().get("observationDatasetId").toString(), program); + // TODO: + // fetch top level dataset obs units + // foreach obs unit in top-level dataset + // for (i=0;i dataset, BrAPITrial experiment, From 7d8fc938155faf452bd20fb5f8b536f22e496377 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 29 May 2024 16:43:38 -0400 Subject: [PATCH 03/28] [BI-2109] incorporated BI-1464 work, used dataset array on experiment --- .../v1/controller/ExperimentController.java | 38 +++++++++- .../constants/BrAPIAdditionalInfoFields.java | 1 + .../request/query/ExperimentExportQuery.java | 2 +- .../BrAPIObservationVariableService.java | 13 ++-- .../brapi/v2/services/BrAPITrialService.java | 30 ++++++-- .../ExperimentObservation.java | 70 +++++++++++++++++-- .../processors/ExperimentProcessor.java | 69 +++++++++++++----- .../org/breedinginsight/model/Dataset.java | 19 +++++ .../breedinginsight/model/DatasetLevel.java | 34 +++++++++ .../model/DatasetMetadata.java | 38 ++++++++++ .../utilities/DatasetUtil.java | 47 +++++++++++++ src/main/resources/application.yml | 2 + 12 files changed, 327 insertions(+), 36 deletions(-) create mode 100644 src/main/java/org/breedinginsight/model/DatasetLevel.java create mode 100644 src/main/java/org/breedinginsight/model/DatasetMetadata.java create mode 100644 src/main/java/org/breedinginsight/utilities/DatasetUtil.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 30a2c1fe4..03a6d0d2b 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -9,6 +9,7 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.request.SubEntityDatasetRequest; @@ -16,6 +17,7 @@ import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.model.Dataset; +import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; @@ -25,6 +27,7 @@ import javax.inject.Inject; import javax.validation.Valid; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -91,10 +94,17 @@ public HttpResponse> getDatasetData( } } + /** + * Creates a sub-entity dataset for a given program and experiment with the specified name and number of repeated measures. + * @param programId The UUID of the program. + * @param experimentId The UUID of the experiment. + * @param datasetRequest The POST body, contains the dataset name and number of repeated measures to create. + * @return An HttpResponse with a Response object containing the newly created Dataset. + */ @Post("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/dataset") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) @Produces(MediaType.APPLICATION_JSON) - public HttpResponse> getDatasetData( + public HttpResponse> createSubEntityDataset( @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, @Body @Valid SubEntityDatasetRequest datasetRequest) { @@ -107,4 +117,30 @@ public HttpResponse> getDatasetData( return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); } } + + /** + * Retrieves the datasets for a given program and experiment. + * + * @param programId The UUID of the program. + * @param experimentId The UUID of the experiment. + * @return An HttpResponse with a Response object containing a list of DatasetMetadata. + * @throws DoesNotExistException if the program does not exist. + * @throws ApiException if an error occurs while retrieving the datasets. + */ + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/datasets") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse>> getDatasets( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId) throws DoesNotExistException, ApiException { + + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response> response = new Response(experimentService.getDatasetsMetadata(programOptional.get(), experimentId)); + return HttpResponse.ok(response); + + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 87fbfdda9..0ac5afea9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -44,6 +44,7 @@ public final class BrAPIAdditionalInfoFields { public static final String ENVIRONMENT_NUMBER = "environmentNumber"; public static final String STUDY_NAME = "studyName"; public static final String OBSERVATION_DATASET_ID = "observationDatasetId"; + public static final String DATASETS = "datasets"; public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown"; public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; public static final String TREATMENTS = "treatments"; diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 8c59be52c..e0d40ba34 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -12,7 +12,7 @@ @ToString public class ExperimentExportQuery { private FileType fileExtension; - private String dataset; + private String dataset; // TODO: rename so it's clear this is dataset NAME, not ID. Also change on frontend. private String environments; @NotNull private boolean includeTimestamps; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java index f9742fb74..cba06dd55 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -31,6 +31,7 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -55,6 +56,8 @@ public BrAPIObservationVariableService( this.referenceSource = referenceSource; } + // TODO: handle non-top level dataset. + // This gets the observation variables for the top-level dataset in an experiment. public List getBrAPIObservationVariablesForExperiment( UUID programId, Optional experimentId, @@ -78,12 +81,10 @@ public List getBrAPIObservationVariablesForExperiment( } BrAPITrial experiment = trialService.getExperiment(program.get(), expId); - if(experiment - .getAdditionalInfo().getAsJsonObject() - .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + if(!experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + String obsDatasetId = DatasetUtil.getDatasetIdByNameFromJson( + experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), + experiment.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); return trialService.getDatasetObsVars(obsDatasetId, program.get()); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 458e486a8..ea136fb60 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapi.v2.services; +import com.google.gson.JsonArray; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -32,6 +33,7 @@ import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.IntOrderComparator; import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; @@ -167,14 +169,12 @@ public DownloadFile exportObservations( log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } - - if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && - experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + String defaultObsLevel = experiment.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString(); + DatasetMetadata datasetMetadata = DatasetUtil.getDatasetByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), defaultObsLevel); + if ((StringUtils.isBlank(params.getDataset()) || defaultObsLevel.equalsIgnoreCase(params.getDataset())) && datasetMetadata != null) { + String obsDatasetId = datasetMetadata.toString(); isDataset = true; - log.debug(logHash + ": fetching dataset observation variables for export"); + log.debug(logHash + ": fetching " + datasetMetadata.getName() + " dataset observation variables for export"); obsVars = getDatasetObsVars(obsDatasetId, program); // make additional columns in the export for each obs variable and obs variable timestamp @@ -331,6 +331,22 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId return dataset; } + /** + * Retrieves the metadata of datasets associated with a program and experiment. + * + * @param program The program object representing the program that the datasets belong to. + * @param experimentId The UUID of the experiment that the datasets are associated with. + * @return A list of DatasetMetadata objects containing the metadata of the datasets. + * @throws DoesNotExistException If the trial does not exist for the program and experimentId combination. + * @throws ApiException If there is an error retrieving the trial or parsing the datasets metadata. + */ + public List getDatasetsMetadata(Program program, UUID experimentId) throws DoesNotExistException, ApiException { + BrAPITrial trial = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); + JsonArray datasetsJson = trial.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + return datasets; + } + public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException { log.debug("creating sub-entity dataset: \"" + request.getName() + "\" for experiment: \"" + experimentId + "\" with: \"" + request.getRepeatedMeasures() + "\" repeated measures."); BrAPITrial experiment = getExperiment(program, experimentId); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 579c34a31..bb756c041 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -33,10 +33,7 @@ import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.model.BrAPIConstants; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.utilities.Utilities; import java.math.BigInteger; @@ -284,7 +281,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" + level.setLevelName(getExpUnit()); level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); @@ -368,6 +365,69 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } + /** + * Constructs a BrAPI sub-observation unit based on the given parameters. Only a subset of the information is + * included as the exp unit level contains all the relevant top-level details and is linked from this unit + * + * Uses only the following methods from ExperimentObservation: + * - getSubObsUnit + * - getSubUnitId + * + * Existing ExpUnit data is from passed in BrAPI object which is looked up based on ObsUnitID, no other values + * in import file are required for creating a sub observation unit dataset + */ + public BrAPIObservationUnit constructBrAPISubObservationUnit( + Program program, + String seqVal, + boolean commit, + BrAPIObservationUnit expUnit, + String referenceSource, + UUID datasetId, + UUID id + ) { + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + if (commit) { + observationUnit.setObservationUnitName(Utilities.appendProgramKey(getSubObsUnit(), program.getKey(), seqVal)); + observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, UUID.fromString(expUnit.getTrialDbId()), + datasetId, UUID.fromString(expUnit.getStudyDbId()), id)); + } else { + observationUnit.setObservationUnitName(getSubObsUnit()); + } + observationUnit.setStudyName(expUnit.getStudyName()); + + observationUnit.setGermplasmName(expUnit.getGermplasmName()); + String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString(); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); + + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + + // observationLevel entry for Sub-Obs Unit + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + level.setLevelName(getSubObsUnit()); + level.setLevelCode(Utilities.appendProgramKey(getSubUnitId(), program.getKey(), seqVal)); + level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); + position.setObservationLevel(level); + + // keep this in case we decide to rename levels in future + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getSubObsUnit()); + + // observationLevelRelationships for top-level Exp Unit linking + List levelRelationships = new ArrayList<>(); + BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); + + // set name without added keys + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + expUnitLevel.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); + expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); + levelRelationships.add(expUnitLevel); + + // TODO: Do replicate and block matter for field book? + + observationUnit.setObservationUnitPosition(position); + return observationUnit; + } + public BrAPIObservation constructBrAPIObservation( String value, String variableName, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 45d26f7cb..8bc3b03bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -65,6 +65,7 @@ import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import tech.tablesaw.api.Table; @@ -132,6 +133,10 @@ public class ExperimentProcessor implements Processor { private Map> observationUnitByNameNoScope = null; private Map> pendingObsUnitByOUId = new HashMap<>(); + // For use when creating sub obs units. + private Map> obsUnitByExpObsUnitId = null; + private Map> subObsUnitBySubObsUnitId = null; + private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method @@ -1132,6 +1137,7 @@ private PendingImportObject getGidPIO(ExperimentObservation impo private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { PendingImportObject pio; + // TODO: should be based on ObsUnitId. String key = createObservationUnitKey(importRow); if (hasAllReferenceUnitIds) { pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); @@ -1148,9 +1154,10 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + // TODO: get dataset id from array on ObsUnit. Lookup by dataset.level=="ExpUnit" from the importRow. + JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + // TODO: handle possible NPE (null case). + datasetId = DatasetUtil.getDatasetByNameFromJson(datasetsJson, importRow.getExpUnit()).getId(); } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); @@ -1280,11 +1287,14 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } }); } + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { PendingImportObject pio; PendingImportObject trialPIO = hasAllReferenceUnitIds ? getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - String name = String.format("Observation Dataset [%s-%s]", + String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); + String name = String.format("%s Observation Dataset [%s-%s]", + datasetName, program.getKey(), trialPIO.getBrAPIObject() .getAdditionalInfo() @@ -1301,7 +1311,24 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr program, trialPIO.getId().toString()); pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + // TODO: remove old id and update code to use datasets array - existing will need migration trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + + JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + // If datasets property does not yet exist, create it + + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + DatasetMetadata dataset = DatasetMetadata.builder() + .name(datasetName) + .id(id) + .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) + .build(); + + log.debug(dataset.getName()); + datasets.add(dataset); + datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); + trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); + if (ImportObjectState.EXISTING == trialPIO.getState()) { trialPIO.setState(ImportObjectState.MUTATED); } @@ -1615,6 +1642,13 @@ private Map> initializeObserva try { List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); if (existingObsUnits.size() == rowByObsUnitId.size()) { existingObsUnits.forEach(brAPIObservationUnit -> { @@ -2013,8 +2047,9 @@ private Map> mapPendingObsDatasetB Map> trialByOUId, Map> obsVarDatasetByName, Map> obsVarDatasetByOUId) { - if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && - trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + if (!trialByOUId.isEmpty() && + !obsVarDatasetByName.isEmpty() && + !trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); } @@ -2029,17 +2064,18 @@ private Map> mapPendingObsDatasetB * @return A map of observation variable dataset objects indexed by dataset name. * @throws InternalServerException If the existing dataset summary is not retrieved from the BrAPI server, or an error occurs during API communication. */ + // TODO: add dataset name or id parameter? This only supports top level for now. private Map> initializeObsVarDatasetForExistingObservationUnits( Map> trialByName, Program program) { Map> obsVarDatasetByName = new HashMap<>(); if (trialByName.size() > 0 && - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialByName.values().iterator().next().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); + !trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + String datasetId = DatasetUtil.getDatasetIdByNameFromJson( + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString() + ); try { List existingDatasets = brAPIListDAO @@ -2067,11 +2103,12 @@ private Map> initializeObsVarDatas Optional> trialPIO = getTrialPIO(experimentImportRows); - if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialPIO.get().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); + // TODO: use value in datasets array instead + if (trialPIO.isPresent() && !trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + String datasetId = DatasetUtil.getDatasetIdByNameFromJson( + trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), + trialPIO.get().getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).toString() + ); try { List existingDatasets = brAPIListDAO .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, diff --git a/src/main/java/org/breedinginsight/model/Dataset.java b/src/main/java/org/breedinginsight/model/Dataset.java index 638c946f1..27823871a 100644 --- a/src/main/java/org/breedinginsight/model/Dataset.java +++ b/src/main/java/org/breedinginsight/model/Dataset.java @@ -1,3 +1,22 @@ + +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.breedinginsight.model; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/src/main/java/org/breedinginsight/model/DatasetLevel.java b/src/main/java/org/breedinginsight/model/DatasetLevel.java new file mode 100644 index 000000000..8da129129 --- /dev/null +++ b/src/main/java/org/breedinginsight/model/DatasetLevel.java @@ -0,0 +1,34 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +public enum DatasetLevel { + EXP_UNIT(0), + SUB_OBS_UNIT(1); + + private final int value; + + DatasetLevel(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/model/DatasetMetadata.java b/src/main/java/org/breedinginsight/model/DatasetMetadata.java new file mode 100644 index 000000000..ca96c6256 --- /dev/null +++ b/src/main/java/org/breedinginsight/model/DatasetMetadata.java @@ -0,0 +1,38 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import java.util.UUID; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@SuperBuilder +@NoArgsConstructor +public class DatasetMetadata { + private String name; + private UUID id; + private DatasetLevel level; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java new file mode 100644 index 000000000..517d5e106 --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java @@ -0,0 +1,47 @@ +package org.breedinginsight.utilities; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.reflect.TypeToken; +import org.brapi.client.v2.JSON; +import org.breedinginsight.model.DatasetMetadata; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; + +public class DatasetUtil { + + public static Type listDatasetType = new TypeToken>() {}.getType(); + public static Gson gson = new JSON().getGson(); + + public static List datasetsFromJson(JsonArray datasetsJsonArray) { + if (datasetsJsonArray != null) { + return gson.fromJson(datasetsJsonArray, listDatasetType); + } + + return new ArrayList<>(); + } + + public static JsonArray jsonArrayFromDatasets(List datasets) { + return gson.toJsonTree(datasets, listDatasetType).getAsJsonArray(); + } + + public static String getDatasetIdByNameFromJson(JsonArray datasetsJsonArray, String datasetName) { + DatasetMetadata dataset = getDatasetByNameFromJson(datasetsJsonArray, datasetName); + if (dataset != null) { + return dataset.getId().toString(); + } + return null; + } + + public static DatasetMetadata getDatasetByNameFromJson(JsonArray datasetsJsonArray, String datasetName) { + List datasets = datasetsFromJson(datasetsJsonArray); + for (DatasetMetadata dataset : datasets) { + if (dataset.getName().equals(datasetName)) { + return dataset; + } + } + return null; + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index aec211b82..019a0336a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -31,6 +31,8 @@ micronaut: mixed: true threshold: '10MB' max-request-size: '100MB' + serialization: + enum-values: true bi: api: version: v1 From 37738f1721458d3468704af25006a9e266bb346b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 31 May 2024 13:58:10 -0400 Subject: [PATCH 04/28] [BI-2109] - fixed enum serialization issue --- src/main/java/org/breedinginsight/model/DatasetLevel.java | 4 ++++ src/main/resources/application.yml | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/model/DatasetLevel.java b/src/main/java/org/breedinginsight/model/DatasetLevel.java index 8da129129..d1f2a0fb6 100644 --- a/src/main/java/org/breedinginsight/model/DatasetLevel.java +++ b/src/main/java/org/breedinginsight/model/DatasetLevel.java @@ -17,8 +17,12 @@ package org.breedinginsight.model; +import com.google.gson.annotations.SerializedName; + public enum DatasetLevel { + @SerializedName("0") EXP_UNIT(0), + @SerializedName("1") SUB_OBS_UNIT(1); private final int value; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 019a0336a..aec211b82 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -31,8 +31,6 @@ micronaut: mixed: true threshold: '10MB' max-request-size: '100MB' - serialization: - enum-values: true bi: api: version: v1 From 9fb3555455be57502b3a9efe88e89eb11c060136 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:12:22 -0400 Subject: [PATCH 05/28] [BI-2109] - fully implemented createSubEntityDataset --- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 25 ++- .../brapi/v2/services/BrAPITrialService.java | 145 ++++++++++++++++-- .../ExperimentObservation.java | 28 ++-- .../utilities/DatasetUtil.java | 16 +- .../breedinginsight/utilities/Utilities.java | 11 ++ 5 files changed, 196 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 7d8fd9651..60c70e11f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -170,7 +170,8 @@ public List getObservationUnitByName(List observat } /** - * Create observation units, mutates brAPIObservationUnitList + * Create observation units with import progress. + * Mutates brAPIObservationUnitList. */ public List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId, ImportUpload upload) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); @@ -190,6 +191,28 @@ public List createBrAPIObservationUnits(List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId) throws ApiException, DoesNotExistException { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); + try { + if (!brAPIObservationUnitList.isEmpty()) { + Callable> postFunction = () -> { + preprocessObservationUnits(brAPIObservationUnitList); + List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); + return processObservationUnitsForCache(ous, program, false); + }; + return programObservationUnitCache.post(programId, postFunction); + } + return new ArrayList<>(); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + public List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { if(observationUnitExternalIds.isEmpty()) { return Collections.emptyList(); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index ea136fb60..5962e458d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -58,6 +58,7 @@ public class BrAPITrialService { private final String referenceSource; private final BrAPITrialDAO trialDAO; private final BrAPIObservationDAO observationDAO; + private final BrAPIObservationUnitDAO observationUnitDAO; private final BrAPIListDAO listDAO; private final TraitService traitService; @@ -72,6 +73,7 @@ public class BrAPITrialService { public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPITrialDAO trialDAO, BrAPIObservationDAO observationDAO, + BrAPIObservationUnitDAO observationUnitDAO, BrAPIListDAO listDAO, TraitService traitService, BrAPIStudyDAO studyDAO, @@ -83,6 +85,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.referenceSource = referenceSource; this.trialDAO = trialDAO; this.observationDAO = observationDAO; + this.observationUnitDAO = observationUnitDAO; this.listDAO = listDAO; this.traitService = traitService; this.studyDAO = studyDAO; @@ -347,18 +350,140 @@ public List getDatasetsMetadata(Program program, UUID experimen return datasets; } - public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException { + public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException { log.debug("creating sub-entity dataset: \"" + request.getName() + "\" for experiment: \"" + experimentId + "\" with: \"" + request.getRepeatedMeasures() + "\" repeated measures."); + UUID subEntityDatasetId = UUID.randomUUID(); + List subObsUnits = new ArrayList<>(); BrAPITrial experiment = getExperiment(program, experimentId); - String a = "b"; - List datasetOUs = ouDAO.getObservationUnitsForDataset(experiment.getAdditionalInfo().get("observationDatasetId").toString(), program); - // TODO: - // fetch top level dataset obs units - // foreach obs unit in top-level dataset - // for (i=0;i expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); + for (BrAPIObservationUnit expUnit : expOUs) { + + // Get environment number from study. + String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() + .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + + for (int i=0; i createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); + + // Update trial with the new dataset to collection in additionalInfo. + DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() + .id(subEntityDatasetId) + .name(request.getName()) + .level(DatasetLevel.SUB_OBS_UNIT) + .build(); + List datasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + datasets.add(subEntityDatasetMetadata); + experiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); + // Ask the DAO to persist the updated trial. + trialDAO.updateBrAPITrial(experiment.getTrialDbId(), experiment, program.getId()); + + // Return the new dataset. + return getDatasetData(program, experimentId, subEntityDatasetId, false); // TODO: stats? + } + + public BrAPIObservationUnit createSubObservationUnit( + String subEntityDatasetName, + String subUnitId, + Program program, + String seqVal, + BrAPIObservationUnit expUnit, + String referenceSource, + UUID datasetId, + UUID id + ) { + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + observationUnit.setObservationUnitName(Utilities.appendProgramKey(subEntityDatasetName, program.getKey(), seqVal)); + + // Build ExternalReferences. + List refs = new ArrayList<>(); + + // Program ref. + Utilities.addReference(refs, program.getId(), referenceSource, ExternalReferenceSource.PROGRAMS); + + // Trial and Study refs can be copied from expUnit to subUnit. + Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.TRIALS) + .ifPresent(refs::add); + Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.STUDIES) + .ifPresent(refs::add); + + // Dataset and ObservationUnit refs are specific to the subUnit. + if (datasetId != null) { + Utilities.addReference(refs, datasetId, referenceSource, ExternalReferenceSource.DATASET); + } + if (id != null) { + Utilities.addReference(refs, id, referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + } + + // Set ExternalReferences. + observationUnit.setExternalReferences(refs); + + // Set Trial. + observationUnit.setTrialName(expUnit.getTrialName()); + observationUnit.setTrialDbId(expUnit.getTrialDbId()); + + // Set Study. + observationUnit.setStudyName(expUnit.getStudyName()); + observationUnit.setStudyDbId(expUnit.getStudyDbId()); + + // Set Germplasm. + observationUnit.setGermplasmName(expUnit.getGermplasmName()); + observationUnit.setGermplasmDbId(expUnit.getGermplasmDbId()); + String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString(); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); + + // keep this in case we decide to rename levels in future + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); + + // Build ObservationUnitPosition. + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + + // ObservationLevel entry for Sub-Obs Unit. + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + level.setLevelName(subEntityDatasetName); + level.setLevelCode(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); + level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); + position.setObservationLevel(level); + + // ObservationLevelRelationships for top-level Exp Unit linking. + List levelRelationships = new ArrayList<>(); + BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); + expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); + expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); + levelRelationships.add(expUnitLevel); + position.setObservationLevelRelationships(levelRelationships); + + // Set ObservationUnitPosition. + observationUnit.setObservationUnitPosition(position); + + // TODO: Do replicate and block matter for field book? + + return observationUnit; } private void addBrAPIObsToRecords( diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index bb756c041..7c78372c7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -243,9 +243,9 @@ public BrAPIListDetails constructDatasetDetails( dataSetDetails.setData(new ArrayList<>()); dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); List refs = new ArrayList<>(); - addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); - addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); - addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); + Utilities.addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); dataSetDetails.setExternalReferences(refs); return dataSetDetails; } @@ -376,6 +376,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( * Existing ExpUnit data is from passed in BrAPI object which is looked up based on ObsUnitID, no other values * in import file are required for creating a sub observation unit dataset */ + // TODO: delete if unused! public BrAPIObservationUnit constructBrAPISubObservationUnit( Program program, String seqVal, @@ -389,6 +390,7 @@ public BrAPIObservationUnit constructBrAPISubObservationUnit( BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); if (commit) { observationUnit.setObservationUnitName(Utilities.appendProgramKey(getSubObsUnit(), program.getKey(), seqVal)); + // TODO: the call to getObsUnitExternalReferences incorrectly uses DbIds, xrefs must be BI generated UUIDs. observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, UUID.fromString(expUnit.getTrialDbId()), datasetId, UUID.fromString(expUnit.getStudyDbId()), id)); } else { @@ -473,21 +475,21 @@ private List getBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { List refs = new ArrayList<>(); - addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); if (trialId != null) { - addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } if (datasetId != null) { - addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); + Utilities.addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); } if (studyId != null) { - addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); + Utilities.addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } if (obsUnitId != null) { - addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); + Utilities.addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); } if (observationId != null) { - addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); + Utilities.addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); } return refs; @@ -513,14 +515,6 @@ private List getObservationExternalReferences( return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, studyId, obsUnitId, observationId); } - - private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { - BrAPIExternalReference reference = new BrAPIExternalReference(); - reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); - reference.setReferenceID(uuid.toString()); - refs.add(reference); - } - public static final class Columns { public static final String GERMPLASM_NAME = "Germplasm Name"; public static final String GERMPLASM_GID = "Germplasm GID"; diff --git a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java index 517d5e106..bd13efa5d 100644 --- a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java +++ b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java @@ -4,6 +4,9 @@ import com.google.gson.JsonArray; import com.google.gson.reflect.TypeToken; import org.brapi.client.v2.JSON; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.model.DatasetLevel; import org.breedinginsight.model.DatasetMetadata; import java.lang.reflect.Type; @@ -44,4 +47,15 @@ public static DatasetMetadata getDatasetByNameFromJson(JsonArray datasetsJsonArr } return null; } -} \ No newline at end of file + + public static DatasetMetadata getTopLevelDatasetFromJson(JsonArray datasetsJsonArray) { + List datasets = datasetsFromJson(datasetsJsonArray); + // Return the top level dataset if it exists, otherwise null. + return datasets.stream().filter(dataset -> dataset.getLevel().equals(DatasetLevel.EXP_UNIT)).findFirst().orElse(null); + } + + public static DatasetMetadata getTopLevelDataset(BrAPITrial experiment) { + return getTopLevelDatasetFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + } + +} diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 2ff850d55..1da02322a 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -181,6 +181,17 @@ public static Optional getExternalReference(List externalReference.getReferenceSource().equals(source)).findFirst(); } + public static Optional getExternalReference(List externalReferences, String referenceSourceBase, ExternalReferenceSource referenceSource) { + return getExternalReference(externalReferences, generateReferenceSource(referenceSourceBase, referenceSource)); + } + + public static void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { + BrAPIExternalReference reference = new BrAPIExternalReference(); + reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); + reference.setReferenceId(uuid.toString()); + refs.add(reference); + } + /** * For a list of items, if the list has only one item, return that item, otherwise return an empty {@link Optional} * @param items {@link List} of items From 0f26eacc83261ac9342e240cd88f07ca88fd125a Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:15:16 -0400 Subject: [PATCH 06/28] [BI-2109] - fixed DatasetLevel serialization in response --- src/main/java/org/breedinginsight/model/DatasetLevel.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/breedinginsight/model/DatasetLevel.java b/src/main/java/org/breedinginsight/model/DatasetLevel.java index d1f2a0fb6..8bad5078f 100644 --- a/src/main/java/org/breedinginsight/model/DatasetLevel.java +++ b/src/main/java/org/breedinginsight/model/DatasetLevel.java @@ -17,8 +17,11 @@ package org.breedinginsight.model; +import com.fasterxml.jackson.annotation.JsonValue; import com.google.gson.annotations.SerializedName; +// The @SerializedName annotations are to make gson serialize correctly in the data access layer. +// The @JsonValue annotation is to make micronaut (via jackson) serialize correctly in the controller response. public enum DatasetLevel { @SerializedName("0") EXP_UNIT(0), @@ -31,6 +34,7 @@ public enum DatasetLevel { this.value = value; } + @JsonValue public int getValue() { return value; } From a4786d7c854f89ae34861a22dad2068f81fb8a49 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:54:32 -0400 Subject: [PATCH 07/28] [BI-2109] - fixed sub observation unit name --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 5962e458d..ecd178df9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -416,7 +416,7 @@ public BrAPIObservationUnit createSubObservationUnit( ) { BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); - observationUnit.setObservationUnitName(Utilities.appendProgramKey(subEntityDatasetName, program.getKey(), seqVal)); + observationUnit.setObservationUnitName(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); // Build ExternalReferences. List refs = new ArrayList<>(); From 086bd9640b81a7c173aed34275de9ed710e7bb65 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:13:01 -0400 Subject: [PATCH 08/28] [BI-2109] - completed sub observation unit position --- .../brapi/v2/services/BrAPITrialService.java | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index ecd178df9..d7d54192d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -461,6 +461,14 @@ public BrAPIObservationUnit createSubObservationUnit( // Build ObservationUnitPosition. BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + // Set subUnit's basic position attributes from expUnit. + position.setEntryType(expUnit.getObservationUnitPosition().getEntryType()); + position.setGeoCoordinates(expUnit.getObservationUnitPosition().getGeoCoordinates()); + position.setPositionCoordinateX(expUnit.getObservationUnitPosition().getPositionCoordinateX()); + position.setPositionCoordinateY(expUnit.getObservationUnitPosition().getPositionCoordinateY()); + position.setPositionCoordinateXType(expUnit.getObservationUnitPosition().getPositionCoordinateXType()); + position.setPositionCoordinateYType(expUnit.getObservationUnitPosition().getPositionCoordinateYType()); + // ObservationLevel entry for Sub-Obs Unit. BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); level.setLevelName(subEntityDatasetName); @@ -468,8 +476,31 @@ public BrAPIObservationUnit createSubObservationUnit( level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); position.setObservationLevel(level); - // ObservationLevelRelationships for top-level Exp Unit linking. + // ObservationLevelRelationships. List levelRelationships = new ArrayList<>(); + // ObservationLevelRelationships for block. + BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> x.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())).findFirst().orElse(null); + if (expBlockLevel != null) { + BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); + blockLevel.setLevelName(expBlockLevel.getLevelName()); + blockLevel.setLevelCode(expBlockLevel.getLevelCode()); + blockLevel.setLevelOrder(expBlockLevel.getLevelOrder()); + levelRelationships.add(blockLevel); + } + // ObservationLevelRelationships for rep. + BrAPIObservationUnitLevelRelationship expRepLevel = expUnit.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> x.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).findFirst().orElse(null); + if (expRepLevel != null) { + BrAPIObservationUnitLevelRelationship repLevel = new BrAPIObservationUnitLevelRelationship(); + repLevel.setLevelName(expRepLevel.getLevelName()); + repLevel.setLevelCode(expRepLevel.getLevelCode()); + repLevel.setLevelOrder(expRepLevel.getLevelOrder()); + levelRelationships.add(repLevel); + } + // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); @@ -481,8 +512,6 @@ public BrAPIObservationUnit createSubObservationUnit( // Set ObservationUnitPosition. observationUnit.setObservationUnitPosition(position); - // TODO: Do replicate and block matter for field book? - return observationUnit; } From c0214a9d7c5e7269934ba896fbf740fd12669ef4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:47:25 -0400 Subject: [PATCH 09/28] [BI-2109] - indexed from 1 --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index d7d54192d..e1a77106d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -369,7 +369,7 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); - for (int i=0; i Date: Mon, 10 Jun 2024 17:13:08 -0400 Subject: [PATCH 10/28] [BI-2109] - added RTK and Treament Factors --- .../brapi/v2/services/BrAPITrialService.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e1a77106d..51fe560c7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -455,8 +455,23 @@ public BrAPIObservationUnit createSubObservationUnit( String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString(); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); - // keep this in case we decide to rename levels in future + // Set treatment factors. + List treatmentFactors = new ArrayList<>(); + for (BrAPIObservationTreatment t : expUnit.getTreatments()) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(t.getFactor()); + treatment.setModality(t.getModality()); + treatmentFactors.add(treatment); + } + observationUnit.setTreatments(treatmentFactors); + + // Put level in additional info: keep this in case we decide to rename levels in future. observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); + // Put RTK in additional info. + String rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK).getAsString(); + if (rtk != null) { + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.RTK, rtk); + } // Build ObservationUnitPosition. BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); From 11dcf7dd8b3eb72195586e2fec8c05e2c59a42e3 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:06:59 -0400 Subject: [PATCH 11/28] [BI-2109] - null handling --- .../brapi/v2/services/BrAPITrialService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 51fe560c7..732233769 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,6 +1,7 @@ package org.breedinginsight.brapi.v2.services; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -452,8 +453,10 @@ public BrAPIObservationUnit createSubObservationUnit( // Set Germplasm. observationUnit.setGermplasmName(expUnit.getGermplasmName()); observationUnit.setGermplasmDbId(expUnit.getGermplasmDbId()); - String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString(); - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); + JsonElement gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID); + if (gid != null) { + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid.getAsString()); + } // Set treatment factors. List treatmentFactors = new ArrayList<>(); @@ -468,9 +471,9 @@ public BrAPIObservationUnit createSubObservationUnit( // Put level in additional info: keep this in case we decide to rename levels in future. observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); // Put RTK in additional info. - String rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK).getAsString(); + JsonElement rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK); if (rtk != null) { - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.RTK, rtk); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.RTK, rtk.getAsString()); } // Build ObservationUnitPosition. From 27db242ec656fd2896baf39065e4b8e4fab27763 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:31:24 -0400 Subject: [PATCH 12/28] [BI-2109] - added id to Dataset --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- src/main/java/org/breedinginsight/model/Dataset.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 732233769..668e40662 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -318,7 +318,7 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); log.debug("building dataset object for dataset: " + datasetId); sortDefaultForObservationUnit(datasetOUs); - Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); + Dataset dataset = new Dataset(datasetId.toString(), experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { Integer ouCount = datasetOUs.size(); Integer obsVarCount = datasetObsVars.size(); diff --git a/src/main/java/org/breedinginsight/model/Dataset.java b/src/main/java/org/breedinginsight/model/Dataset.java index 27823871a..9cee40591 100644 --- a/src/main/java/org/breedinginsight/model/Dataset.java +++ b/src/main/java/org/breedinginsight/model/Dataset.java @@ -29,6 +29,7 @@ @JsonInclude(JsonInclude.Include.NON_ABSENT) public class Dataset { + public String id; public String experimentId; public JsonObject additionalInfo; public List data; @@ -50,10 +51,12 @@ public enum DatasetStat { } public Dataset( + String id, String experimentId, List data, List observationUnits, List observationVariables) { + this.id = id; this.experimentId = experimentId; this.additionalInfo = new JsonObject(); this.data = data; From 23efe19f8d21aa38336ea6cdd5edeeca1b79952b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 09:38:20 -0400 Subject: [PATCH 13/28] [BI-2109] - removed unused method --- .../ExperimentObservation.java | 65 ------------------- 1 file changed, 65 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 7c78372c7..741670723 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -365,71 +365,6 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } - /** - * Constructs a BrAPI sub-observation unit based on the given parameters. Only a subset of the information is - * included as the exp unit level contains all the relevant top-level details and is linked from this unit - * - * Uses only the following methods from ExperimentObservation: - * - getSubObsUnit - * - getSubUnitId - * - * Existing ExpUnit data is from passed in BrAPI object which is looked up based on ObsUnitID, no other values - * in import file are required for creating a sub observation unit dataset - */ - // TODO: delete if unused! - public BrAPIObservationUnit constructBrAPISubObservationUnit( - Program program, - String seqVal, - boolean commit, - BrAPIObservationUnit expUnit, - String referenceSource, - UUID datasetId, - UUID id - ) { - - BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); - if (commit) { - observationUnit.setObservationUnitName(Utilities.appendProgramKey(getSubObsUnit(), program.getKey(), seqVal)); - // TODO: the call to getObsUnitExternalReferences incorrectly uses DbIds, xrefs must be BI generated UUIDs. - observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, UUID.fromString(expUnit.getTrialDbId()), - datasetId, UUID.fromString(expUnit.getStudyDbId()), id)); - } else { - observationUnit.setObservationUnitName(getSubObsUnit()); - } - observationUnit.setStudyName(expUnit.getStudyName()); - - observationUnit.setGermplasmName(expUnit.getGermplasmName()); - String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString(); - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); - - BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); - - // observationLevel entry for Sub-Obs Unit - BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName(getSubObsUnit()); - level.setLevelCode(Utilities.appendProgramKey(getSubUnitId(), program.getKey(), seqVal)); - level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); - position.setObservationLevel(level); - - // keep this in case we decide to rename levels in future - observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getSubObsUnit()); - - // observationLevelRelationships for top-level Exp Unit linking - List levelRelationships = new ArrayList<>(); - BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - - // set name without added keys - expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); - expUnitLevel.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); - expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); - levelRelationships.add(expUnitLevel); - - // TODO: Do replicate and block matter for field book? - - observationUnit.setObservationUnitPosition(position); - return observationUnit; - } - public BrAPIObservation constructBrAPIObservation( String value, String variableName, From 2467cb0c3c4472a2bfb5b68b63bebdabc4c6ba0d Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:34:55 -0400 Subject: [PATCH 14/28] [BI-2109] - cleaned up --- .../v2/model/request/query/ExperimentExportQuery.java | 2 +- .../v2/services/BrAPIObservationVariableService.java | 8 ++++---- .../importer/services/processors/ExperimentProcessor.java | 4 ---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index e0d40ba34..11683e4d7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -12,7 +12,7 @@ @ToString public class ExperimentExportQuery { private FileType fileExtension; - private String dataset; // TODO: rename so it's clear this is dataset NAME, not ID. Also change on frontend. + private String dataset; // TODO: rename so it's clear if this is dataset NAME or ID. Also change on frontend. private String environments; @NotNull private boolean includeTimestamps; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java index cba06dd55..96a21cd14 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -56,7 +56,7 @@ public BrAPIObservationVariableService( this.referenceSource = referenceSource; } - // TODO: handle non-top level dataset. + // TODO: support sub-entity datasets. // This gets the observation variables for the top-level dataset in an experiment. public List getBrAPIObservationVariablesForExperiment( UUID programId, @@ -82,9 +82,9 @@ public List getBrAPIObservationVariablesForExperiment( BrAPITrial experiment = trialService.getExperiment(program.get(), expId); if(!experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - String obsDatasetId = DatasetUtil.getDatasetIdByNameFromJson( - experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), - experiment.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString()); + String obsDatasetId = DatasetUtil + .getTopLevelDatasetFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)) + .getId().toString(); return trialService.getDatasetObsVars(obsDatasetId, program.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b57b856f1..54b1d3020 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -133,10 +133,6 @@ public class ExperimentProcessor implements Processor { private Map> observationUnitByNameNoScope = null; private Map> pendingObsUnitByOUId = new HashMap<>(); - // For use when creating sub obs units. - private Map> obsUnitByExpObsUnitId = null; - private Map> subObsUnitBySubObsUnitId = null; - private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method From 2f02edbca30ba954b2986d930d7853047d90d0fd Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:44:44 -0400 Subject: [PATCH 15/28] [BI-2109] - improved comments --- .../brapi/v2/services/BrAPITrialService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 5c95f595b..f7823b023 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -252,7 +252,7 @@ public DownloadFile exportObservations( List> rows = entry.getValue(); sortDefaultForExportRows(rows); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); - // TODO: remove hardcoded datasetName, use observation level. + // TODO: [BI-2183] remove hardcoded datasetName, use observation level. String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); @@ -274,7 +274,7 @@ public DownloadFile exportObservations( StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - // TODO: remove hardcoded datasetName, use observation level. + // TODO: [BI-2183] remove hardcoded datasetName, use observation level. String fileName = makeFileName(experiment, program, envFilenameFragment, "Observation Dataset") + fileType.getExtension(); downloadFile = new DownloadFile(fileName, streamedFile); } @@ -391,7 +391,7 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt List createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); - // Update trial with the new dataset to collection in additionalInfo. + // Add the new dataset metadata to the datasets array in the trial's additionalInfo. DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() .id(subEntityDatasetId) .name(request.getName()) @@ -404,7 +404,7 @@ public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEnt trialDAO.updateBrAPITrial(experiment.getTrialDbId(), experiment, program.getId()); // Return the new dataset. - return getDatasetData(program, experimentId, subEntityDatasetId, false); // TODO: stats? + return getDatasetData(program, experimentId, subEntityDatasetId, false); } public BrAPIObservationUnit createSubObservationUnit( From 7090b60977d084d6ff485afcc0511946e8aa5cfd Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:53:55 -0400 Subject: [PATCH 16/28] [BI-2109] - handled null case --- .../importer/services/processors/ExperimentProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 54b1d3020..a381925ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1144,7 +1144,6 @@ private PendingImportObject getGidPIO(ExperimentObservation impo private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { PendingImportObject pio; - // TODO: should be based on ObsUnitId. String key = createObservationUnitKey(importRow); if (hasAllReferenceUnitIds) { pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); @@ -1161,10 +1160,11 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { - // TODO: get dataset id from array on ObsUnit. Lookup by dataset.level=="ExpUnit" from the importRow. JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); - // TODO: handle possible NPE (null case). - datasetId = DatasetUtil.getDatasetByNameFromJson(datasetsJson, importRow.getExpUnit()).getId(); + DatasetMetadata dataset = DatasetUtil.getDatasetByNameFromJson(datasetsJson, importRow.getExpUnit()); + if (dataset != null) { + datasetId = dataset.getId(); + } } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); From 4226ef2fe35e00ab4d578cc007f138eb05f01356 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:02:41 -0400 Subject: [PATCH 17/28] [BI-2109] - cleaned up ExperimentProcessor.java changes --- .../services/processors/ExperimentProcessor.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a381925ca..c208279a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1660,13 +1660,6 @@ private Map> initializeObserva try { List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); - // TODO: grab from externalReferences - /* - observationUnitByObsUnitId = existingObsUnits.stream() - .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, - (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); - */ - String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); if (existingObsUnits.size() == rowByObsUnitId.size()) { existingObsUnits.forEach(brAPIObservationUnit -> { @@ -2121,12 +2114,10 @@ private Map> initializeObsVarDatas Optional> trialPIO = getTrialPIO(experimentImportRows); - // TODO: use value in datasets array instead if (trialPIO.isPresent() && !trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { - String datasetId = DatasetUtil.getDatasetIdByNameFromJson( - trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), - trialPIO.get().getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).toString() - ); + String datasetId = DatasetUtil.getTopLevelDatasetFromJson( + trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS) + ).getId().toString(); try { List existingDatasets = brAPIListDAO .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, From cbaef191d16101e0887778af271466ca7c5e35a7 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:42:16 -0400 Subject: [PATCH 18/28] [BI-2109] - cleaned up --- .../importer/services/processors/ExperimentProcessor.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c208279a8..c0f5c4c1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1310,9 +1310,7 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr PendingImportObject pio; PendingImportObject trialPIO = hasAllReferenceUnitIds ? getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); - String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); - String name = String.format("%s Observation Dataset [%s-%s]", - datasetName, + String name = String.format("Observation Dataset [%s-%s]", program.getKey(), trialPIO.getBrAPIObject() .getAdditionalInfo() @@ -1329,12 +1327,10 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr program, trialPIO.getId().toString()); pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - // TODO: remove old id and update code to use datasets array - existing will need migration - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); // If datasets property does not yet exist, create it - + String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); List datasets = DatasetUtil.datasetsFromJson(datasetsJson); DatasetMetadata dataset = DatasetMetadata.builder() .name(datasetName) From 24fe163801c1c938129e2c7def6f6fea92024bfe Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:17:39 -0400 Subject: [PATCH 19/28] [BI-2109] - fixed several tests --- .../constants/BrAPIAdditionalInfoFields.java | 1 + .../utilities/DatasetUtil.java | 4 ++++ ...vationLevelsControllerIntegrationTest.java | 2 +- ...ervationUnitControllerIntegrationTest.java | 2 +- ...ObservationsControllerIntegrationTest.java | 2 +- ...BrAPIStudiesControllerIntegrationTest.java | 2 +- ...tionVariableControllerIntegrationTest.java | 6 +++--- .../ExperimentControllerIntegrationTest.java | 2 +- .../importer/ExperimentFileImportTest.java | 20 +++++++++++-------- 9 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 0ac5afea9..6ea30cd20 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -43,6 +43,7 @@ public final class BrAPIAdditionalInfoFields { public static final String EXPERIMENT_NUMBER = "experimentNumber"; public static final String ENVIRONMENT_NUMBER = "environmentNumber"; public static final String STUDY_NAME = "studyName"; + @Deprecated public static final String OBSERVATION_DATASET_ID = "observationDatasetId"; public static final String DATASETS = "datasets"; public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown"; diff --git a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java index bd13efa5d..f64bbadc5 100644 --- a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java +++ b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java @@ -39,6 +39,10 @@ public static String getDatasetIdByNameFromJson(JsonArray datasetsJsonArray, Str } public static DatasetMetadata getDatasetByNameFromJson(JsonArray datasetsJsonArray, String datasetName) { + // Short-circuiting null check. + if (datasetsJsonArray == null || datasetName == null) { + return null; + } List datasets = datasetsFromJson(datasetsJsonArray); for (DatasetMetadata dataset : datasets) { if (dataset.getName().equals(datasetName)) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 85f6c6773..e38ab05fc 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -249,7 +249,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 3a3d8ffa3..bdfe304c0 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -381,7 +381,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 16d7261fb..5b4e562a6 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -337,7 +337,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java index ac276003c..faee28da4 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -335,7 +335,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 58c20194f..79056913e 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -156,7 +156,7 @@ void setup() throws Exception { List germplasm = createGermplasm(1); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); - newReference.setReferenceID(program.getId().toString()); + newReference.setReferenceId(program.getId().toString()); germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); @@ -344,7 +344,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { @@ -363,7 +363,7 @@ private List createGermplasm(int numToCreate) { List externalRef = new ArrayList<>(); BrAPIExternalReference testReference = new BrAPIExternalReference(); testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); - testReference.setReferenceID(UUID.randomUUID().toString()); + testReference.setReferenceId(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); germplasm.add(testGermplasm); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 630c29c85..e7c8a11d6 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -482,7 +482,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 48bffd4c2..2dda40e60 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -446,10 +446,11 @@ public void importNewExpWithObsVar() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); @@ -691,10 +692,11 @@ public void importNewObsVarExistingOu() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); @@ -742,10 +744,11 @@ public void importNewObsVarByObsUnitId() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); @@ -807,10 +810,11 @@ public void importNewObservationDataByObsUnitId(boolean commit) { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); From 5dd733c65a5f16d935be470f9f3aaf4efaee0a09 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:19:14 -0400 Subject: [PATCH 20/28] [BI-2109] - fixed bug --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index f7823b023..d3306de28 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -178,7 +178,7 @@ public DownloadFile exportObservations( String defaultObsLevel = experiment.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString(); DatasetMetadata datasetMetadata = DatasetUtil.getDatasetByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), defaultObsLevel); if ((StringUtils.isBlank(params.getDataset()) || defaultObsLevel.equalsIgnoreCase(params.getDataset())) && datasetMetadata != null) { - String obsDatasetId = datasetMetadata.toString(); + String obsDatasetId = datasetMetadata.getId().toString(); isDataset = true; log.debug(logHash + ": fetching " + datasetMetadata.getName() + " dataset observation variables for export"); obsVars = getDatasetObsVars(obsDatasetId, program); From 0d847055fe277bbe6a336c8032ffd957697142f9 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 24 Jun 2024 17:20:05 -0400 Subject: [PATCH 21/28] [BI-2109] - fixed failing tests --- .../experimentObservation/ExperimentObservation.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 741670723..0f2596e6f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -281,7 +281,12 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName(getExpUnit()); + // If expUnit is null, a validation error will be produced later on. + if (getExpUnit() != null) + { + // TODO: [BI-2219] BJTS only accepts hardcoded levels, need to handle dynamic levels. + level.setLevelName(getExpUnit().toLowerCase()); // HACK: toLowerCase() is needed to match BJTS hardcoded levels. + } level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); From abf014d617ac9269cde6c2d515c0b2bf005eaf56 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:19:54 -0400 Subject: [PATCH 22/28] [BI-2109] - implemented download by dataset --- .../brapi/v2/dao/BrAPIObservationDAO.java | 18 +++++++++ .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 15 +++++++ .../request/query/ExperimentExportQuery.java | 2 +- .../brapi/v2/services/BrAPITrialService.java | 40 +++++++------------ 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 6eb4c2761..92b45633c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -205,6 +205,24 @@ public List getObservationsByObservationUnitsAndVariables(Coll .collect(Collectors.toList()); } + public List getObservationsByObservationUnits(Collection ouDbIds, Program program) throws ApiException { + if(ouDbIds.isEmpty()) { + return Collections.emptyList(); + } + return getProgramObservations(program.getId()).values().stream() + .filter(o -> ouDbIds.contains(o.getObservationUnitDbId())) + .collect(Collectors.toList()); + } + + public List getObservationsByObservationUnitsAndStudies(Collection ouDbIds, Collection studyDbIds, Program program) throws ApiException { + if(ouDbIds.isEmpty()) { + return Collections.emptyList(); + } + return getProgramObservations(program.getId()).values().stream() + .filter(o -> ouDbIds.contains(o.getObservationUnitDbId()) && studyDbIds.contains(o.getStudyDbId())) + .collect(Collectors.toList()); + } + @NotNull private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 60c70e11f..aeb919a2f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -254,6 +254,21 @@ public List getObservationUnitsForDataset(@NotNull String .collect(Collectors.toList()); } + public List getObservationUnitsForDatasetAndEnvs(@NotNull String datasetId, Collection envIds, @NotNull Program program) throws ApiException { + String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); + String studyReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); + return getProgramObservationUnits(program.getId()).values().stream() + .filter(ou -> { + Optional datasetExRef = Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource); + Optional studyExRef = Utilities.getExternalReference(ou.getExternalReferences(), studyReferenceSource); + return Boolean.logicalAnd( + datasetExRef.map(x -> x.getReferenceId().equals(datasetId)).orElse(false), + studyExRef.map(x -> envIds.contains(x.getReferenceId())).orElse(false) + ); + }) + .collect(Collectors.toList()); + } + // Note: does not use cache, impractical to implement all search parameters client-side. public List getObservationUnits(Program program, Optional observationUnitId, diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 11683e4d7..01782c319 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -12,7 +12,7 @@ @ToString public class ExperimentExportQuery { private FileType fileExtension; - private String dataset; // TODO: rename so it's clear if this is dataset NAME or ID. Also change on frontend. + private String datasetId; private String environments; @NotNull private boolean includeTimestamps; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index d3306de28..bcd82e694 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -133,7 +133,6 @@ public DownloadFile exportObservations( String logHash = UUID.randomUUID().toString(); log.debug(logHash + ": exporting experiment: "+experimentId+", params: " + params); DownloadFile downloadFile; - boolean isDataset = false; List dataset = new ArrayList<>(); List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); @@ -161,42 +160,32 @@ public DownloadFile exportObservations( } expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); - // get the OUs for the requested environments + // Get the OUs for the requested environments. log.debug(logHash + ": fetching OUs for export"); List ous = new ArrayList<>(); Map ouByOUDbId = new HashMap<>(); try { - for (BrAPIStudy study: expStudies) { - List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); - studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); - ous.addAll(studyOUs); + if (requestedEnvIds.isEmpty()) { + ous.addAll(ouDAO.getObservationUnitsForDataset(params.getDatasetId(), program)); + } else { + ous.addAll(ouDAO.getObservationUnitsForDatasetAndEnvs(params.getDatasetId(), requestedEnvIds, program)); } + ous.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); } catch (ApiException err) { log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } - String defaultObsLevel = experiment.getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString(); - DatasetMetadata datasetMetadata = DatasetUtil.getDatasetByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), defaultObsLevel); - if ((StringUtils.isBlank(params.getDataset()) || defaultObsLevel.equalsIgnoreCase(params.getDataset())) && datasetMetadata != null) { - String obsDatasetId = datasetMetadata.getId().toString(); - isDataset = true; - log.debug(logHash + ": fetching " + datasetMetadata.getName() + " dataset observation variables for export"); - obsVars = getDatasetObsVars(obsDatasetId, program); - - // make additional columns in the export for each obs variable and obs variable timestamp - addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + if (params.getDatasetId() != null) { + log.debug(logHash + ": fetching " + params.getDatasetId() + " dataset observation variables for export"); + obsVars = getDatasetObsVars(params.getDatasetId(), program); + // Make additional columns in the export for each obs variable and obs variable timestamp. + addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); } - // make export rows from any observations - if (isDataset) { - log.debug(logHash + ": fetching observations for export"); - dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); - } - if (!requestedEnvIds.isEmpty()) { - log.debug(logHash + ": filtering observations to only requested environments for export"); - dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); - } + // Make export rows from any observations. + log.debug(logHash + ": fetching observations for export"); + dataset = observationDAO.getObservationsByObservationUnits(ouByOUDbId.keySet(), program); log.debug(logHash + ": fetching program's germplasm for export"); List programGermplasm = germplasmDAO.getGermplasmsByDBID(ouByOUDbId.values().stream().map(BrAPIObservationUnit::getGermplasmDbId).collect(Collectors.toList()), program.getId()); @@ -217,7 +206,6 @@ public DownloadFile exportObservations( programGermplasmByDbId ); - // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { for (BrAPIObservationUnit ou: ous) { From 913cc143844eaa5df96b3bdf113f65b257c74cb4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:31:54 -0400 Subject: [PATCH 23/28] [BI-2109] - commented test --- .../brapi/v2/ExperimentControllerIntegrationTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index e7c8a11d6..1b1f76d03 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -220,6 +220,9 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE String envs = numberOfEnvsRequested == 1 ? envIds.get(0) : String.join(",", envIds); envParam = "environments=" + envs; } + // TODO:Get available datasets for experiment. + // Get datasetId. + // Send datasetId in request. Flowable> call = client.exchange( GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) @@ -259,7 +262,6 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps); - } // Remove temp directory after each test run. FileUtils.deleteDirectory(new File(tempDir)); From 2e0825d1135d7d3930cadd7fc3be7f2ce40643eb Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 27 Jun 2024 17:09:22 -0400 Subject: [PATCH 24/28] [BI-2109] - fixed failing test --- .../v2/ExperimentControllerIntegrationTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 1b1f76d03..0bfa099bb 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -22,6 +22,7 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -74,6 +75,8 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { @Inject private OntologyService ontologyService; @Inject + private BrAPITrialService experimentService; + @Inject private BrAPIGermplasmDAO germplasmDAO; @Inject @@ -220,12 +223,11 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE String envs = numberOfEnvsRequested == 1 ? envIds.get(0) : String.join(",", envIds); envParam = "environments=" + envs; } - // TODO:Get available datasets for experiment. - // Get datasetId. - // Send datasetId in request. + // Get datasetId to include in export request. + String datasetId = experimentService.getDatasetsMetadata(program, UUID.fromString(experimentId)).stream().findFirst().get().getId().toString(); Flowable> call = client.exchange( - GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", - program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) + GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s&datasetId=%s", + program.getId().toString(), experimentId, includeTimestamps, envParam, extension, datasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse response = call.blockingFirst(); From 8730d74c48815c40fbb205d1c91bc163a6d05017 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:23:44 -0400 Subject: [PATCH 25/28] [BI-2109] - added sub-entity download test --- .../ExperimentControllerIntegrationTest.java | 123 ++++++++++++++++-- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 0bfa099bb..e29cdb8e6 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -14,6 +14,7 @@ import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; @@ -36,6 +37,7 @@ import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.FileUtil; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -212,6 +214,11 @@ void setup() throws Exception { "true,XLSX,2", "false,XLSX,2",}) @SneakyThrows void downloadDatasets(boolean includeTimestamps, String extension, int numberOfEnvsRequested) { + // How many columns are expected in the output? + int expectedColNumber = columns.size(); + if (includeTimestamps) { + expectedColNumber += traits.size(); + } // Temporary directory to extract zip into, test will clean up after use. String tempDir = "./zip_temp_dir/"; // If more than 1 envId is sent as a query param, a zip file is expected response. @@ -256,19 +263,106 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE List> filteredRows = rows.stream() .filter(row -> file.getName().contains(row.get(ExperimentObservation.Columns.ENV).toString())) .collect(Collectors.toList()); - parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps); + parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber); } } else { assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; - parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps); + parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber); } // Remove temp directory after each test run. FileUtils.deleteDirectory(new File(tempDir)); } + /** + * Tests creating and subsequently downloading a sub-entity dataset. + * It also ensures no regressions with the top-level dataset download are introduced by the sub-entity features. + */ + @ParameterizedTest + @CsvSource(value = {"CSV", "XLSX", "XLS"}) + @SneakyThrows + void downloadSubEntityDataset(String extension) { + + // Create sub-entity dataset. + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), experimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":3}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + byte[].class); + HttpResponse postResponse = postCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, postResponse.getStatus()); + + // Get top-level datasetId to include in export request. + BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(experimentId), false); + String topLevelDatasetId = DatasetUtil.getTopLevelDataset(experiment).getId().toString(); + Flowable> topLevelExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), experimentId, extension, topLevelDatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse topLevelResponse = topLevelExportCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, topLevelResponse.getStatus()); + + // Assert file format fidelity + Map mediaTypeByExtension = new HashMap<>(); + mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); + mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); + mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); + String downloadMediaType = topLevelResponse.getHeaders().getContentType().orElseThrow(Exception::new); + assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); + + // Check file contents. + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(topLevelResponse.body())); + parseAndCheck(bodyStream, extension, false, rows, false, 25); + + // Make sub-entity dataset export request. + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + Flowable> plantExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), experimentId, extension, plantDatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plantResponse = plantExportCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, plantResponse.getStatus()); + + // Assert file format fidelity + assertEquals(mediaTypeByExtension.get(extension), plantResponse.getHeaders().getContentType().orElseThrow(Exception::new)); + + // The expected contents of the exported Plant dataset (3 sub-obs units for each top-level unit were requested). + List> plantRows = buildSubEntityRows(rows, "Plant", 3); + + // Check file contents. + ByteArrayInputStream plantBodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); + // TODO: find a way to build expected rows for plant dataset. + parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); + } + + private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { + List> plantRows = new ArrayList<>(); + for (Map row : topLevelRows) { + for (Integer i=1; i<=repeatedMeasures; i++) { + // Deep copy map entries. + Map plantRow = new HashMap<>(row); + + plantRow.put("Exp Unit", entityName); + plantRow.put("Exp Unit ID", i.toString()); + plantRow.remove("tt_test_1"); + plantRow.remove("tt_test_2"); + plantRows.add(plantRow); + } + } + return plantRows; + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -368,7 +462,8 @@ private void parseAndCheck(InputStream stream, String extension, boolean requestEnv, List> rows, - boolean includeTimestamps) throws ParsingException { + boolean includeTimestamps, + Integer expectedColNumber) throws ParsingException { Table download = Table.create(); if (extension.equals("CSV")) { download = FileUtil.parseTableFromCsv(stream); @@ -376,8 +471,9 @@ private void parseAndCheck(InputStream stream, if (extension.equals("XLS") || extension.equals("XLSX")) { download = FileUtil.parseTableFromExcel(stream, 0); } + // Assert import/export fidelity and presence of observation units in export - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension, expectedColNumber); } private void checkDownloadTable( @@ -385,15 +481,12 @@ private void checkDownloadTable( List> requestedImportRows, Table table, boolean includeTimestamps, - String extension) { + String extension, + Integer expectedColNumber) { // Filename is correct: _Observation Dataset [-]__ List expectedEnvNames = requestedImportRows.stream() .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()).collect(Collectors.toList()); - // All columns included - Integer expectedColNumber = columns.size(); - if (includeTimestamps) { - expectedColNumber += traits.size(); - } + assertEquals(expectedColNumber, table.columnCount()); // Check that requested envs are present. @@ -406,15 +499,19 @@ private void checkDownloadTable( Row downloadRow = table.row(rowNum); // sort order is not guaranteed to be th same as import, so find import row for corresponding export row - // by first matching environment and GID + // by first matching environment, GID and Exp Unit ID matchingImportRow = requestedImportRows.stream().filter(row -> { String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; + String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && - row.get(env).equals(downloadRow.getString(env)); + row.get(env).equals(downloadRow.getString(env)) && + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); } else { - return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)); + return row.get(gid).equals(downloadRow.getString(gid)) && + row.get(env).equals(downloadRow.getString(env)) && + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); From c64b1297479ffc857559f2750e593bb22d632a03 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 3 Jul 2024 11:25:18 -0400 Subject: [PATCH 26/28] [BI-2109] - cleaned up comment --- .../brapi/v2/ExperimentControllerIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index e29cdb8e6..80c281c39 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -342,7 +342,6 @@ void downloadSubEntityDataset(String extension) { // Check file contents. ByteArrayInputStream plantBodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); - // TODO: find a way to build expected rows for plant dataset. parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); } From dd82dd7da4ae41299719e803cc8a72610daeeb2f Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:42:08 -0400 Subject: [PATCH 27/28] [BI-2109] - added workaround until BI-2219 is implemented --- .../brapi/v2/services/BrAPITrialService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index bcd82e694..192739ccb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -479,7 +479,8 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevel entry for Sub-Obs Unit. BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName(subEntityDatasetName); + // TODO: consider removing toLowerCase() after BI-2219 is implemented. + level.setLevelName(subEntityDatasetName.toLowerCase()); level.setLevelCode(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); position.setObservationLevel(level); @@ -510,7 +511,8 @@ public BrAPIObservationUnit createSubObservationUnit( } // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); - expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + // TODO: consider removing toLowerCase() after BI-2219 is implemented. + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); From a82118f27e584c695ad0d85172d3a422c382e7f5 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:43:38 -0400 Subject: [PATCH 28/28] [BI-2109] - added 404 response --- .../api/v1/controller/ExperimentController.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 03a6d0d2b..77214c9cf 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -109,8 +109,12 @@ public HttpResponse> createSubEntityDataset( @PathVariable("experimentId") UUID experimentId, @Body @Valid SubEntityDatasetRequest datasetRequest) { try { - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); - Response response = new Response(experimentService.createSubEntityDataset(program, experimentId, datasetRequest)); + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response response = new Response(experimentService.createSubEntityDataset(programOptional.get(), experimentId, datasetRequest)); return HttpResponse.ok(response); } catch (Exception e){ log.info(e.getMessage());