diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java b/src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java new file mode 100644 index 000000000..c45cfae31 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/SubEntityDatasetRequest.java @@ -0,0 +1,38 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.model.v1.request; + +import io.micronaut.core.annotation.Introspected; +import lombok.*; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +@Introspected +public class SubEntityDatasetRequest { + @NotBlank + private String name; + @NotNull + private int repeatedMeasures; +} diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 017197219..77214c9cf 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -9,20 +9,25 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.api.model.v1.request.SubEntityDatasetRequest; import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.model.Dataset; +import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; import javax.validation.Valid; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -89,4 +94,57 @@ public HttpResponse> getDatasetData( } } + /** + * Creates a sub-entity dataset for a given program and experiment with the specified name and number of repeated measures. + * @param programId The UUID of the program. + * @param experimentId The UUID of the experiment. + * @param datasetRequest The POST body, contains the dataset name and number of repeated measures to create. + * @return An HttpResponse with a Response object containing the newly created Dataset. + */ + @Post("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/dataset") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse> createSubEntityDataset( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId, + @Body @Valid SubEntityDatasetRequest datasetRequest) { + try { + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response response = new Response(experimentService.createSubEntityDataset(programOptional.get(), experimentId, datasetRequest)); + return HttpResponse.ok(response); + } catch (Exception e){ + log.info(e.getMessage()); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } + } + + /** + * Retrieves the datasets for a given program and experiment. + * + * @param programId The UUID of the program. + * @param experimentId The UUID of the experiment. + * @return An HttpResponse with a Response object containing a list of DatasetMetadata. + * @throws DoesNotExistException if the program does not exist. + * @throws ApiException if an error occurs while retrieving the datasets. + */ + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/datasets") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse>> getDatasets( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId) throws DoesNotExistException, ApiException { + + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response> response = new Response(experimentService.getDatasetsMetadata(programOptional.get(), experimentId)); + return HttpResponse.ok(response); + + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 87fbfdda9..6ea30cd20 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -43,7 +43,9 @@ public final class BrAPIAdditionalInfoFields { public static final String EXPERIMENT_NUMBER = "experimentNumber"; public static final String ENVIRONMENT_NUMBER = "environmentNumber"; public static final String STUDY_NAME = "studyName"; + @Deprecated public static final String OBSERVATION_DATASET_ID = "observationDatasetId"; + public static final String DATASETS = "datasets"; public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown"; public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; public static final String TREATMENTS = "treatments"; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 6eb4c2761..92b45633c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -205,6 +205,24 @@ public List getObservationsByObservationUnitsAndVariables(Coll .collect(Collectors.toList()); } + public List getObservationsByObservationUnits(Collection ouDbIds, Program program) throws ApiException { + if(ouDbIds.isEmpty()) { + return Collections.emptyList(); + } + return getProgramObservations(program.getId()).values().stream() + .filter(o -> ouDbIds.contains(o.getObservationUnitDbId())) + .collect(Collectors.toList()); + } + + public List getObservationsByObservationUnitsAndStudies(Collection ouDbIds, Collection studyDbIds, Program program) throws ApiException { + if(ouDbIds.isEmpty()) { + return Collections.emptyList(); + } + return getProgramObservations(program.getId()).values().stream() + .filter(o -> ouDbIds.contains(o.getObservationUnitDbId()) && studyDbIds.contains(o.getStudyDbId())) + .collect(Collectors.toList()); + } + @NotNull private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 7d8fd9651..aeb919a2f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -170,7 +170,8 @@ public List getObservationUnitByName(List observat } /** - * Create observation units, mutates brAPIObservationUnitList + * Create observation units with import progress. + * Mutates brAPIObservationUnitList. */ public List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId, ImportUpload upload) throws ApiException, DoesNotExistException { Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); @@ -190,6 +191,28 @@ public List createBrAPIObservationUnits(List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId) throws ApiException, DoesNotExistException { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); + try { + if (!brAPIObservationUnitList.isEmpty()) { + Callable> postFunction = () -> { + preprocessObservationUnits(brAPIObservationUnitList); + List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); + return processObservationUnitsForCache(ous, program, false); + }; + return programObservationUnitCache.post(programId, postFunction); + } + return new ArrayList<>(); + } catch (Exception e) { + throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + } + } + public List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { if(observationUnitExternalIds.isEmpty()) { return Collections.emptyList(); @@ -231,6 +254,21 @@ public List getObservationUnitsForDataset(@NotNull String .collect(Collectors.toList()); } + public List getObservationUnitsForDatasetAndEnvs(@NotNull String datasetId, Collection envIds, @NotNull Program program) throws ApiException { + String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); + String studyReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); + return getProgramObservationUnits(program.getId()).values().stream() + .filter(ou -> { + Optional datasetExRef = Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource); + Optional studyExRef = Utilities.getExternalReference(ou.getExternalReferences(), studyReferenceSource); + return Boolean.logicalAnd( + datasetExRef.map(x -> x.getReferenceId().equals(datasetId)).orElse(false), + studyExRef.map(x -> envIds.contains(x.getReferenceId())).orElse(false) + ); + }) + .collect(Collectors.toList()); + } + // Note: does not use cache, impractical to implement all search parameters client-side. public List getObservationUnits(Program program, Optional observationUnitId, diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java index 8c59be52c..01782c319 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -12,7 +12,7 @@ @ToString public class ExperimentExportQuery { private FileType fileExtension; - private String dataset; + private String datasetId; private String environments; @NotNull private boolean includeTimestamps; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java index f9742fb74..96a21cd14 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -31,6 +31,7 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -55,6 +56,8 @@ public BrAPIObservationVariableService( this.referenceSource = referenceSource; } + // TODO: support sub-entity datasets. + // This gets the observation variables for the top-level dataset in an experiment. public List getBrAPIObservationVariablesForExperiment( UUID programId, Optional experimentId, @@ -78,12 +81,10 @@ public List getBrAPIObservationVariablesForExperiment( } BrAPITrial experiment = trialService.getExperiment(program.get(), expId); - if(experiment - .getAdditionalInfo().getAsJsonObject() - .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + if(!experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + String obsDatasetId = DatasetUtil + .getTopLevelDatasetFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)) + .getId().toString(); return trialService.getDatasetObsVars(obsDatasetId, program.get()); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index dd9bde018..192739ccb 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,7 @@ package org.breedinginsight.brapi.v2.services; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.github.filosganga.geogson.model.Coordinates; import com.github.filosganga.geogson.model.positions.SinglePosition; import com.google.gson.JsonObject; @@ -16,6 +18,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.*; +import org.breedinginsight.api.model.v1.request.SubEntityDatasetRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; @@ -33,6 +36,7 @@ import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.IntOrderComparator; import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; @@ -57,6 +61,7 @@ public class BrAPITrialService { private final String referenceSource; private final BrAPITrialDAO trialDAO; private final BrAPIObservationDAO observationDAO; + private final BrAPIObservationUnitDAO observationUnitDAO; private final BrAPIListDAO listDAO; private final TraitService traitService; @@ -71,6 +76,7 @@ public class BrAPITrialService { public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPITrialDAO trialDAO, BrAPIObservationDAO observationDAO, + BrAPIObservationUnitDAO observationUnitDAO, BrAPIListDAO listDAO, TraitService traitService, BrAPIStudyDAO studyDAO, @@ -82,6 +88,7 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.referenceSource = referenceSource; this.trialDAO = trialDAO; this.observationDAO = observationDAO; + this.observationUnitDAO = observationUnitDAO; this.listDAO = listDAO; this.traitService = traitService; this.studyDAO = studyDAO; @@ -126,7 +133,6 @@ public DownloadFile exportObservations( String logHash = UUID.randomUUID().toString(); log.debug(logHash + ": exporting experiment: "+experimentId+", params: " + params); DownloadFile downloadFile; - boolean isDataset = false; List dataset = new ArrayList<>(); List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); @@ -154,44 +160,32 @@ public DownloadFile exportObservations( } expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); - // get the OUs for the requested environments + // Get the OUs for the requested environments. log.debug(logHash + ": fetching OUs for export"); List ous = new ArrayList<>(); Map ouByOUDbId = new HashMap<>(); try { - for (BrAPIStudy study: expStudies) { - List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); - studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); - ous.addAll(studyOUs); + if (requestedEnvIds.isEmpty()) { + ous.addAll(ouDAO.getObservationUnitsForDataset(params.getDatasetId(), program)); + } else { + ous.addAll(ouDAO.getObservationUnitsForDatasetAndEnvs(params.getDatasetId(), requestedEnvIds, program)); } + ous.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); } catch (ApiException err) { log.error(logHash + ": Error fetching observation units for a study by its DbId" + Utilities.generateApiExceptionLogMessage(err), err); } + if (params.getDatasetId() != null) { + log.debug(logHash + ": fetching " + params.getDatasetId() + " dataset observation variables for export"); + obsVars = getDatasetObsVars(params.getDatasetId(), program); - if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && - experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - isDataset = true; - log.debug(logHash + ": fetching dataset observation variables for export"); - obsVars = getDatasetObsVars(obsDatasetId, program); - - // make additional columns in the export for each obs variable and obs variable timestamp + // Make additional columns in the export for each obs variable and obs variable timestamp. addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); - } - // make export rows from any observations - if (isDataset) { - log.debug(logHash + ": fetching observations for export"); - dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); - } - if (!requestedEnvIds.isEmpty()) { - log.debug(logHash + ": filtering observations to only requested environments for export"); - dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); - } + // Make export rows from any observations. + log.debug(logHash + ": fetching observations for export"); + dataset = observationDAO.getObservationsByObservationUnits(ouByOUDbId.keySet(), program); log.debug(logHash + ": fetching program's germplasm for export"); List programGermplasm = germplasmDAO.getGermplasmsByDBID(ouByOUDbId.values().stream().map(BrAPIObservationUnit::getGermplasmDbId).collect(Collectors.toList()), program.getId()); @@ -212,7 +206,6 @@ public DownloadFile exportObservations( programGermplasmByDbId ); - // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { for (BrAPIObservationUnit ou: ous) { @@ -247,7 +240,8 @@ public DownloadFile exportObservations( List> rows = entry.getValue(); sortDefaultForExportRows(rows); StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); - String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); + // TODO: [BI-2183] remove hardcoded datasetName, use observation level. + String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName(), "Observation Dataset") + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); } @@ -268,7 +262,8 @@ public DownloadFile exportObservations( StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); - String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); + // TODO: [BI-2183] remove hardcoded datasetName, use observation level. + String fileName = makeFileName(experiment, program, envFilenameFragment, "Observation Dataset") + fileType.getExtension(); downloadFile = new DownloadFile(fileName, streamedFile); } @@ -301,19 +296,19 @@ private StreamedFile zipFiles(List files) throws IOException { return new StreamedFile(in, new MediaType(MediaType.APPLICATION_OCTET_STREAM)); } - public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, Boolean stats) throws ApiException, DoesNotExistException { - log.debug("fetching dataset: " + datsetId + " for experiment: " + experimentId + ". including stats: " + stats); - log.debug("fetching observationUnits for dataset: " + datsetId); - List datasetOUs = ouDAO.getObservationUnitsForDataset(datsetId.toString(), program); - log.debug("fetching dataset variables dataset: " + datsetId); - List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); + public Dataset getDatasetData(Program program, UUID experimentId, UUID datasetId, Boolean stats) throws ApiException, DoesNotExistException { + log.debug("fetching dataset: " + datasetId + " for experiment: " + experimentId + ". including stats: " + stats); + log.debug("fetching observationUnits for dataset: " + datasetId); + List datasetOUs = ouDAO.getObservationUnitsForDataset(datasetId.toString(), program); + log.debug("fetching dataset variables dataset: " + datasetId); + List datasetObsVars = getDatasetObsVars(datasetId.toString(), program); List ouDbIds = datasetOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); List obsVarDbIds = datasetObsVars.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()); - log.debug("fetching observations for dataset: " + datsetId); + log.debug("fetching observations for dataset: " + datasetId); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); - log.debug("building dataset object for dataset: " + datsetId); + log.debug("building dataset object for dataset: " + datasetId); sortDefaultForObservationUnit(datasetOUs); - Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); + Dataset dataset = new Dataset(datasetId.toString(), experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { Integer ouCount = datasetOUs.size(); Integer obsVarCount = datasetObsVars.size(); @@ -330,6 +325,206 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, return dataset; } + /** + * Retrieves the metadata of datasets associated with a program and experiment. + * + * @param program The program object representing the program that the datasets belong to. + * @param experimentId The UUID of the experiment that the datasets are associated with. + * @return A list of DatasetMetadata objects containing the metadata of the datasets. + * @throws DoesNotExistException If the trial does not exist for the program and experimentId combination. + * @throws ApiException If there is an error retrieving the trial or parsing the datasets metadata. + */ + public List getDatasetsMetadata(Program program, UUID experimentId) throws DoesNotExistException, ApiException { + BrAPITrial trial = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); + JsonArray datasetsJson = trial.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + return datasets; + } + + public Dataset createSubEntityDataset(Program program, UUID experimentId, SubEntityDatasetRequest request) throws ApiException, DoesNotExistException { + log.debug("creating sub-entity dataset: \"" + request.getName() + "\" for experiment: \"" + experimentId + "\" with: \"" + request.getRepeatedMeasures() + "\" repeated measures."); + UUID subEntityDatasetId = UUID.randomUUID(); + List subObsUnits = new ArrayList<>(); + BrAPITrial experiment = getExperiment(program, experimentId); + // Get top level dataset ObservationUnits. + DatasetMetadata topLevelDataset = DatasetUtil.getTopLevelDataset(experiment); + if (topLevelDataset == null) { + log.error("Experiment {} has no top level dataset.", experiment.getTrialDbId()); + throw new RuntimeException("Cannot create sub-entity dataset for experiment without top level dataset."); + } + + List expOUs = ouDAO.getObservationUnitsForDataset(topLevelDataset.getId().toString(), program); + for (BrAPIObservationUnit expUnit : expOUs) { + + // Get environment number from study. + String envSeqValue = studyDAO.getStudyByDbId(expUnit.getStudyDbId(), program).orElseThrow() + .getAdditionalInfo().get(BrAPIAdditionalInfoFields.ENVIRONMENT_NUMBER).getAsString(); + + for (int i=1; i<=request.getRepeatedMeasures(); i++) { + // Create subObsUnit and add to list. + subObsUnits.add( + createSubObservationUnit( + request.getName(), + Integer.toString(i), + program, + envSeqValue, + expUnit, + this.referenceSource, + subEntityDatasetId, + UUID.randomUUID() + ) + ); + } + } + + List createdObservationUnits = observationUnitDAO.createBrAPIObservationUnits(subObsUnits, program.getId()); + + // Add the new dataset metadata to the datasets array in the trial's additionalInfo. + DatasetMetadata subEntityDatasetMetadata = DatasetMetadata.builder() + .id(subEntityDatasetId) + .name(request.getName()) + .level(DatasetLevel.SUB_OBS_UNIT) + .build(); + List datasets = DatasetUtil.datasetsFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + datasets.add(subEntityDatasetMetadata); + experiment.getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, DatasetUtil.jsonArrayFromDatasets(datasets)); + // Ask the DAO to persist the updated trial. + trialDAO.updateBrAPITrial(experiment.getTrialDbId(), experiment, program.getId()); + + // Return the new dataset. + return getDatasetData(program, experimentId, subEntityDatasetId, false); + } + + public BrAPIObservationUnit createSubObservationUnit( + String subEntityDatasetName, + String subUnitId, + Program program, + String seqVal, + BrAPIObservationUnit expUnit, + String referenceSource, + UUID datasetId, + UUID id + ) { + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + observationUnit.setObservationUnitName(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); + + // Build ExternalReferences. + List refs = new ArrayList<>(); + + // Program ref. + Utilities.addReference(refs, program.getId(), referenceSource, ExternalReferenceSource.PROGRAMS); + + // Trial and Study refs can be copied from expUnit to subUnit. + Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.TRIALS) + .ifPresent(refs::add); + Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.STUDIES) + .ifPresent(refs::add); + + // Dataset and ObservationUnit refs are specific to the subUnit. + if (datasetId != null) { + Utilities.addReference(refs, datasetId, referenceSource, ExternalReferenceSource.DATASET); + } + if (id != null) { + Utilities.addReference(refs, id, referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + } + + // Set ExternalReferences. + observationUnit.setExternalReferences(refs); + + // Set Trial. + observationUnit.setTrialName(expUnit.getTrialName()); + observationUnit.setTrialDbId(expUnit.getTrialDbId()); + + // Set Study. + observationUnit.setStudyName(expUnit.getStudyName()); + observationUnit.setStudyDbId(expUnit.getStudyDbId()); + + // Set Germplasm. + observationUnit.setGermplasmName(expUnit.getGermplasmName()); + observationUnit.setGermplasmDbId(expUnit.getGermplasmDbId()); + JsonElement gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID); + if (gid != null) { + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid.getAsString()); + } + + // Set treatment factors. + List treatmentFactors = new ArrayList<>(); + for (BrAPIObservationTreatment t : expUnit.getTreatments()) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(t.getFactor()); + treatment.setModality(t.getModality()); + treatmentFactors.add(treatment); + } + observationUnit.setTreatments(treatmentFactors); + + // Put level in additional info: keep this in case we decide to rename levels in future. + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, subEntityDatasetName); + // Put RTK in additional info. + JsonElement rtk = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.RTK); + if (rtk != null) { + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.RTK, rtk.getAsString()); + } + + // Build ObservationUnitPosition. + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + + // Set subUnit's basic position attributes from expUnit. + position.setEntryType(expUnit.getObservationUnitPosition().getEntryType()); + position.setGeoCoordinates(expUnit.getObservationUnitPosition().getGeoCoordinates()); + position.setPositionCoordinateX(expUnit.getObservationUnitPosition().getPositionCoordinateX()); + position.setPositionCoordinateY(expUnit.getObservationUnitPosition().getPositionCoordinateY()); + position.setPositionCoordinateXType(expUnit.getObservationUnitPosition().getPositionCoordinateXType()); + position.setPositionCoordinateYType(expUnit.getObservationUnitPosition().getPositionCoordinateYType()); + + // ObservationLevel entry for Sub-Obs Unit. + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + // TODO: consider removing toLowerCase() after BI-2219 is implemented. + level.setLevelName(subEntityDatasetName.toLowerCase()); + level.setLevelCode(Utilities.appendProgramKey(subUnitId, program.getKey(), seqVal)); + level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); + position.setObservationLevel(level); + + // ObservationLevelRelationships. + List levelRelationships = new ArrayList<>(); + // ObservationLevelRelationships for block. + BrAPIObservationUnitLevelRelationship expBlockLevel = expUnit.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> x.getLevelName().equals(BrAPIConstants.REPLICATE.getValue())).findFirst().orElse(null); + if (expBlockLevel != null) { + BrAPIObservationUnitLevelRelationship blockLevel = new BrAPIObservationUnitLevelRelationship(); + blockLevel.setLevelName(expBlockLevel.getLevelName()); + blockLevel.setLevelCode(expBlockLevel.getLevelCode()); + blockLevel.setLevelOrder(expBlockLevel.getLevelOrder()); + levelRelationships.add(blockLevel); + } + // ObservationLevelRelationships for rep. + BrAPIObservationUnitLevelRelationship expRepLevel = expUnit.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> x.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).findFirst().orElse(null); + if (expRepLevel != null) { + BrAPIObservationUnitLevelRelationship repLevel = new BrAPIObservationUnitLevelRelationship(); + repLevel.setLevelName(expRepLevel.getLevelName()); + repLevel.setLevelCode(expRepLevel.getLevelCode()); + repLevel.setLevelOrder(expRepLevel.getLevelOrder()); + levelRelationships.add(repLevel); + } + // ObservationLevelRelationships for top-level Exp Unit linking. + BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); + // TODO: consider removing toLowerCase() after BI-2219 is implemented. + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString().toLowerCase()); + String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); + expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); + expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); + levelRelationships.add(expUnitLevel); + position.setObservationLevelRelationships(levelRelationships); + + // Set ObservationUnitPosition. + observationUnit.setObservationUnitPosition(position); + + return observationUnit; + } + private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, @@ -564,12 +759,14 @@ private void addObsVarColumns( } } } - private String makeFileName(BrAPITrial experiment, Program program, String envName) { - // _Observation Dataset__ + + private String makeFileName(BrAPITrial experiment, Program program, String envName, String datasetName) { + // ___ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - String unsafeName = String.format("%s_Observation Dataset_%s_%s", + String unsafeName = String.format("%s_%s_%s_%s", Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), + datasetName, Utilities.removeProgramKeyAndUnknownAdditionalData(envName, program.getKey()), timestamp); // Make file name safe for all platforms. diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 579c34a31..0f2596e6f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -33,10 +33,7 @@ import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.model.BrAPIConstants; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.utilities.Utilities; import java.math.BigInteger; @@ -246,9 +243,9 @@ public BrAPIListDetails constructDatasetDetails( dataSetDetails.setData(new ArrayList<>()); dataSetDetails.putAdditionalInfoItem("datasetType", "observationDataset"); List refs = new ArrayList<>(); - addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); - addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); - addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); + Utilities.addReference(refs, program.getId(), referenceSourceBase, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, UUID.fromString(trialId), referenceSourceBase, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, datasetId, referenceSourceBase, ExternalReferenceSource.DATASET); dataSetDetails.setExternalReferences(refs); return dataSetDetails; } @@ -284,7 +281,12 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" + // If expUnit is null, a validation error will be produced later on. + if (getExpUnit() != null) + { + // TODO: [BI-2219] BJTS only accepts hardcoded levels, need to handle dynamic levels. + level.setLevelName(getExpUnit().toLowerCase()); // HACK: toLowerCase() is needed to match BJTS hardcoded levels. + } level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); @@ -413,21 +415,21 @@ private List getBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID datasetId, UUID studyId, UUID obsUnitId, UUID observationId) { List refs = new ArrayList<>(); - addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); + Utilities.addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); if (trialId != null) { - addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); + Utilities.addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } if (datasetId != null) { - addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); + Utilities.addReference(refs, datasetId, referenceSourceBaseName, ExternalReferenceSource.DATASET); } if (studyId != null) { - addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); + Utilities.addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } if (obsUnitId != null) { - addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); + Utilities.addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); } if (observationId != null) { - addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); + Utilities.addReference(refs, observationId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATIONS); } return refs; @@ -453,14 +455,6 @@ private List getObservationExternalReferences( return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, studyId, obsUnitId, observationId); } - - private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { - BrAPIExternalReference reference = new BrAPIExternalReference(); - reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); - reference.setReferenceID(uuid.toString()); - refs.add(reference); - } - public static final class Columns { public static final String GERMPLASM_NAME = "Germplasm Name"; public static final String GERMPLASM_GID = "Germplasm GID"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 95e196c5b..c0f5c4c1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -65,6 +65,7 @@ import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import tech.tablesaw.api.Table; @@ -1159,9 +1160,11 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { - datasetId = UUID.fromString(trialPIO.getBrAPIObject() - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + DatasetMetadata dataset = DatasetUtil.getDatasetByNameFromJson(datasetsJson, importRow.getExpUnit()); + if (dataset != null) { + datasetId = dataset.getId(); + } } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); @@ -1302,6 +1305,7 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } }); } + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { PendingImportObject pio; PendingImportObject trialPIO = hasAllReferenceUnitIds ? @@ -1323,7 +1327,22 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr program, trialPIO.getId().toString()); pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); - trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + + JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + // If datasets property does not yet exist, create it + String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + DatasetMetadata dataset = DatasetMetadata.builder() + .name(datasetName) + .id(id) + .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) + .build(); + + log.debug(dataset.getName()); + datasets.add(dataset); + datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); + trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); + if (ImportObjectState.EXISTING == trialPIO.getState()) { trialPIO.setState(ImportObjectState.MUTATED); } @@ -2035,8 +2054,9 @@ private Map> mapPendingObsDatasetB Map> trialByOUId, Map> obsVarDatasetByName, Map> obsVarDatasetByOUId) { - if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && - trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + if (!trialByOUId.isEmpty() && + !obsVarDatasetByName.isEmpty() && + !trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); } @@ -2051,17 +2071,18 @@ private Map> mapPendingObsDatasetB * @return A map of observation variable dataset objects indexed by dataset name. * @throws InternalServerException If the existing dataset summary is not retrieved from the BrAPI server, or an error occurs during API communication. */ + // TODO: add dataset name or id parameter? This only supports top level for now. private Map> initializeObsVarDatasetForExistingObservationUnits( Map> trialByName, Program program) { Map> obsVarDatasetByName = new HashMap<>(); if (trialByName.size() > 0 && - trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialByName.values().iterator().next().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); + !trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + String datasetId = DatasetUtil.getDatasetIdByNameFromJson( + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS), + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL).getAsString() + ); try { List existingDatasets = brAPIListDAO @@ -2089,11 +2110,10 @@ private Map> initializeObsVarDatas Optional> trialPIO = getTrialPIO(experimentImportRows); - if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = trialPIO.get().getBrAPIObject() - .getAdditionalInfo() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) - .getAsString(); + if (trialPIO.isPresent() && !trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) { + String datasetId = DatasetUtil.getTopLevelDatasetFromJson( + trialPIO.get().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS) + ).getId().toString(); try { List existingDatasets = brAPIListDAO .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, diff --git a/src/main/java/org/breedinginsight/model/Dataset.java b/src/main/java/org/breedinginsight/model/Dataset.java index 638c946f1..9cee40591 100644 --- a/src/main/java/org/breedinginsight/model/Dataset.java +++ b/src/main/java/org/breedinginsight/model/Dataset.java @@ -1,3 +1,22 @@ + +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.breedinginsight.model; import com.fasterxml.jackson.annotation.JsonInclude; @@ -10,6 +29,7 @@ @JsonInclude(JsonInclude.Include.NON_ABSENT) public class Dataset { + public String id; public String experimentId; public JsonObject additionalInfo; public List data; @@ -31,10 +51,12 @@ public enum DatasetStat { } public Dataset( + String id, String experimentId, List data, List observationUnits, List observationVariables) { + this.id = id; this.experimentId = experimentId; this.additionalInfo = new JsonObject(); this.data = data; diff --git a/src/main/java/org/breedinginsight/model/DatasetLevel.java b/src/main/java/org/breedinginsight/model/DatasetLevel.java new file mode 100644 index 000000000..8bad5078f --- /dev/null +++ b/src/main/java/org/breedinginsight/model/DatasetLevel.java @@ -0,0 +1,42 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.gson.annotations.SerializedName; + +// The @SerializedName annotations are to make gson serialize correctly in the data access layer. +// The @JsonValue annotation is to make micronaut (via jackson) serialize correctly in the controller response. +public enum DatasetLevel { + @SerializedName("0") + EXP_UNIT(0), + @SerializedName("1") + SUB_OBS_UNIT(1); + + private final int value; + + DatasetLevel(int value) { + this.value = value; + } + + @JsonValue + public int getValue() { + return value; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/model/DatasetMetadata.java b/src/main/java/org/breedinginsight/model/DatasetMetadata.java new file mode 100644 index 000000000..ca96c6256 --- /dev/null +++ b/src/main/java/org/breedinginsight/model/DatasetMetadata.java @@ -0,0 +1,38 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import java.util.UUID; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@SuperBuilder +@NoArgsConstructor +public class DatasetMetadata { + private String name; + private UUID id; + private DatasetLevel level; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java new file mode 100644 index 000000000..f64bbadc5 --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java @@ -0,0 +1,65 @@ +package org.breedinginsight.utilities; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.reflect.TypeToken; +import org.brapi.client.v2.JSON; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.model.DatasetLevel; +import org.breedinginsight.model.DatasetMetadata; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; + +public class DatasetUtil { + + public static Type listDatasetType = new TypeToken>() {}.getType(); + public static Gson gson = new JSON().getGson(); + + public static List datasetsFromJson(JsonArray datasetsJsonArray) { + if (datasetsJsonArray != null) { + return gson.fromJson(datasetsJsonArray, listDatasetType); + } + + return new ArrayList<>(); + } + + public static JsonArray jsonArrayFromDatasets(List datasets) { + return gson.toJsonTree(datasets, listDatasetType).getAsJsonArray(); + } + + public static String getDatasetIdByNameFromJson(JsonArray datasetsJsonArray, String datasetName) { + DatasetMetadata dataset = getDatasetByNameFromJson(datasetsJsonArray, datasetName); + if (dataset != null) { + return dataset.getId().toString(); + } + return null; + } + + public static DatasetMetadata getDatasetByNameFromJson(JsonArray datasetsJsonArray, String datasetName) { + // Short-circuiting null check. + if (datasetsJsonArray == null || datasetName == null) { + return null; + } + List datasets = datasetsFromJson(datasetsJsonArray); + for (DatasetMetadata dataset : datasets) { + if (dataset.getName().equals(datasetName)) { + return dataset; + } + } + return null; + } + + public static DatasetMetadata getTopLevelDatasetFromJson(JsonArray datasetsJsonArray) { + List datasets = datasetsFromJson(datasetsJsonArray); + // Return the top level dataset if it exists, otherwise null. + return datasets.stream().filter(dataset -> dataset.getLevel().equals(DatasetLevel.EXP_UNIT)).findFirst().orElse(null); + } + + public static DatasetMetadata getTopLevelDataset(BrAPITrial experiment) { + return getTopLevelDatasetFromJson(experiment.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS)); + } + +} diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 2231f3a16..3d9d6d04c 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -191,6 +191,17 @@ public static Optional getExternalReference(List externalReference.getReferenceSource().equals(source)).findFirst(); } + public static Optional getExternalReference(List externalReferences, String referenceSourceBase, ExternalReferenceSource referenceSource) { + return getExternalReference(externalReferences, generateReferenceSource(referenceSourceBase, referenceSource)); + } + + public static void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { + BrAPIExternalReference reference = new BrAPIExternalReference(); + reference.setReferenceSource(String.format("%s/%s", referenceBaseNameSource, refSourceName.getName())); + reference.setReferenceId(uuid.toString()); + refs.add(reference); + } + /** * For a list of items, if the list has only one item, return that item, otherwise return an empty {@link Optional} * @param items {@link List} of items diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 85f6c6773..e38ab05fc 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -249,7 +249,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 3a3d8ffa3..bdfe304c0 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -381,7 +381,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 16d7261fb..5b4e562a6 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -337,7 +337,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java index ac276003c..faee28da4 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -335,7 +335,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 58c20194f..79056913e 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -156,7 +156,7 @@ void setup() throws Exception { List germplasm = createGermplasm(1); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); - newReference.setReferenceID(program.getId().toString()); + newReference.setReferenceId(program.getId().toString()); germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); @@ -344,7 +344,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } private List createGermplasm(int numToCreate) { @@ -363,7 +363,7 @@ private List createGermplasm(int numToCreate) { List externalRef = new ArrayList<>(); BrAPIExternalReference testReference = new BrAPIExternalReference(); testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); - testReference.setReferenceID(UUID.randomUUID().toString()); + testReference.setReferenceId(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); germplasm.add(testGermplasm); diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index 630c29c85..80c281c39 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -14,6 +14,7 @@ import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; @@ -22,6 +23,7 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -35,6 +37,7 @@ import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.FileUtil; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -74,6 +77,8 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { @Inject private OntologyService ontologyService; @Inject + private BrAPITrialService experimentService; + @Inject private BrAPIGermplasmDAO germplasmDAO; @Inject @@ -209,6 +214,11 @@ void setup() throws Exception { "true,XLSX,2", "false,XLSX,2",}) @SneakyThrows void downloadDatasets(boolean includeTimestamps, String extension, int numberOfEnvsRequested) { + // How many columns are expected in the output? + int expectedColNumber = columns.size(); + if (includeTimestamps) { + expectedColNumber += traits.size(); + } // Temporary directory to extract zip into, test will clean up after use. String tempDir = "./zip_temp_dir/"; // If more than 1 envId is sent as a query param, a zip file is expected response. @@ -220,9 +230,11 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE String envs = numberOfEnvsRequested == 1 ? envIds.get(0) : String.join(",", envIds); envParam = "environments=" + envs; } + // Get datasetId to include in export request. + String datasetId = experimentService.getDatasetsMetadata(program, UUID.fromString(experimentId)).stream().findFirst().get().getId().toString(); Flowable> call = client.exchange( - GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", - program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) + GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s&datasetId=%s", + program.getId().toString(), experimentId, includeTimestamps, envParam, extension, datasetId)) .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class ); HttpResponse response = call.blockingFirst(); @@ -251,20 +263,105 @@ void downloadDatasets(boolean includeTimestamps, String extension, int numberOfE List> filteredRows = rows.stream() .filter(row -> file.getName().contains(row.get(ExperimentObservation.Columns.ENV).toString())) .collect(Collectors.toList()); - parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps); + parseAndCheck(fileStream, extension, true, filteredRows, includeTimestamps, expectedColNumber); } } else { assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); // All (both) rows when 0 or 2 envs sent, first row when 1 env sent as query param. List> filteredRows = numberOfEnvsRequested == 1 ? List.of(rows.get(0)) : rows; - parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps); - + parseAndCheck(bodyStream, extension, numberOfEnvsRequested > 0, filteredRows, includeTimestamps, expectedColNumber); } // Remove temp directory after each test run. FileUtils.deleteDirectory(new File(tempDir)); } + /** + * Tests creating and subsequently downloading a sub-entity dataset. + * It also ensures no regressions with the top-level dataset download are introduced by the sub-entity features. + */ + @ParameterizedTest + @CsvSource(value = {"CSV", "XLSX", "XLS"}) + @SneakyThrows + void downloadSubEntityDataset(String extension) { + + // Create sub-entity dataset. + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/experiments/%s/dataset", + program.getId().toString(), experimentId), + "{\"name\":\"Plant\",\"repeatedMeasures\":3}") + .cookie(new NettyCookie("phylo-token", "test-registered-user")), + byte[].class); + HttpResponse postResponse = postCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, postResponse.getStatus()); + + // Get top-level datasetId to include in export request. + BrAPITrial experiment = experimentService.getTrialDataByUUID(program.getId(), UUID.fromString(experimentId), false); + String topLevelDatasetId = DatasetUtil.getTopLevelDataset(experiment).getId().toString(); + Flowable> topLevelExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), experimentId, extension, topLevelDatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse topLevelResponse = topLevelExportCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, topLevelResponse.getStatus()); + + // Assert file format fidelity + Map mediaTypeByExtension = new HashMap<>(); + mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); + mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); + mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); + String downloadMediaType = topLevelResponse.getHeaders().getContentType().orElseThrow(Exception::new); + assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); + + // Check file contents. + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(topLevelResponse.body())); + parseAndCheck(bodyStream, extension, false, rows, false, 25); + + // Make sub-entity dataset export request. + String plantDatasetId = DatasetUtil.getDatasetIdByNameFromJson(experiment.getAdditionalInfo().getAsJsonArray("datasets"), "Plant"); + Flowable> plantExportCall = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?all=true&includeTimestamps=false&fileExtension=%s&datasetId=%s", + program.getId().toString(), experimentId, extension, plantDatasetId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse plantResponse = plantExportCall.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, plantResponse.getStatus()); + + // Assert file format fidelity + assertEquals(mediaTypeByExtension.get(extension), plantResponse.getHeaders().getContentType().orElseThrow(Exception::new)); + + // The expected contents of the exported Plant dataset (3 sub-obs units for each top-level unit were requested). + List> plantRows = buildSubEntityRows(rows, "Plant", 3); + + // Check file contents. + ByteArrayInputStream plantBodyStream = new ByteArrayInputStream(Objects.requireNonNull(plantResponse.body())); + parseAndCheck(plantBodyStream, extension, false, plantRows, false, 23); + } + + private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { + List> plantRows = new ArrayList<>(); + for (Map row : topLevelRows) { + for (Integer i=1; i<=repeatedMeasures; i++) { + // Deep copy map entries. + Map plantRow = new HashMap<>(row); + + plantRow.put("Exp Unit", entityName); + plantRow.put("Exp Unit ID", i.toString()); + plantRow.remove("tt_test_1"); + plantRow.remove("tt_test_2"); + plantRows.add(plantRow); + } + } + return plantRows; + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -364,7 +461,8 @@ private void parseAndCheck(InputStream stream, String extension, boolean requestEnv, List> rows, - boolean includeTimestamps) throws ParsingException { + boolean includeTimestamps, + Integer expectedColNumber) throws ParsingException { Table download = Table.create(); if (extension.equals("CSV")) { download = FileUtil.parseTableFromCsv(stream); @@ -372,8 +470,9 @@ private void parseAndCheck(InputStream stream, if (extension.equals("XLS") || extension.equals("XLSX")) { download = FileUtil.parseTableFromExcel(stream, 0); } + // Assert import/export fidelity and presence of observation units in export - checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension, expectedColNumber); } private void checkDownloadTable( @@ -381,15 +480,12 @@ private void checkDownloadTable( List> requestedImportRows, Table table, boolean includeTimestamps, - String extension) { + String extension, + Integer expectedColNumber) { // Filename is correct: _Observation Dataset [-]__ List expectedEnvNames = requestedImportRows.stream() .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()).collect(Collectors.toList()); - // All columns included - Integer expectedColNumber = columns.size(); - if (includeTimestamps) { - expectedColNumber += traits.size(); - } + assertEquals(expectedColNumber, table.columnCount()); // Check that requested envs are present. @@ -402,15 +498,19 @@ private void checkDownloadTable( Row downloadRow = table.row(rowNum); // sort order is not guaranteed to be th same as import, so find import row for corresponding export row - // by first matching environment and GID + // by first matching environment, GID and Exp Unit ID matchingImportRow = requestedImportRows.stream().filter(row -> { String gid = ExperimentObservation.Columns.GERMPLASM_GID; String env = ExperimentObservation.Columns.ENV; + String expUnitId = ExperimentObservation.Columns.EXP_UNIT_ID; if (extension.equalsIgnoreCase(FileType.CSV.getName())) { return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && - row.get(env).equals(downloadRow.getString(env)); + row.get(env).equals(downloadRow.getString(env)) && + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); } else { - return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)); + return row.get(gid).equals(downloadRow.getString(gid)) && + row.get(env).equals(downloadRow.getString(env)) && + row.get(expUnitId).equals(downloadRow.getObject(expUnitId).toString()); } }).findAny(); assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); @@ -482,7 +582,7 @@ private String getEnvId(JsonObject result, int index) { .get("brAPIObject").getAsJsonObject() .get("externalReferences").getAsJsonArray() .get(2).getAsJsonObject() - .get("referenceID").getAsString(); + .get("referenceId").getAsString(); } } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 48bffd4c2..2dda40e60 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -446,10 +446,11 @@ public void importNewExpWithObsVar() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); @@ -691,10 +692,11 @@ public void importNewObsVarExistingOu() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); @@ -742,10 +744,11 @@ public void importNewObsVarByObsUnitId() { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); @@ -807,10 +810,11 @@ public void importNewObservationDataByObsUnitId(boolean commit) { JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); + JsonArray datasets = row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertFalse(datasets.isEmpty()); + assertTrue(importTestUtils.UUID_REGEX.matcher(datasets.get(0).getAsJsonObject().get("id").getAsString()).matches()); assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString());