diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java b/src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java new file mode 100644 index 000000000..fa5929d71 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/query/DatasetParams.java @@ -0,0 +1,49 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.model.v1.request.query; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.breedinginsight.api.v1.controller.metadata.SortOrder; +import org.breedinginsight.brapps.importer.model.exports.FileType; + +import javax.validation.constraints.Positive; + + +@Getter +@Setter +@Introspected +@NoArgsConstructor +public class DatasetParams implements FileDownloadParams { + + private static String DEFAULT_FILETYPE_NAME = FileType.XLSX.getName(); + + private String fileExtension; + private String dataset; + private String environments; + @JsonIgnore + private boolean includeTimestamps; + + public String getDefaultFileTypeName() { + return DEFAULT_FILETYPE_NAME; + } + +} diff --git a/src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java b/src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java new file mode 100644 index 000000000..622aa3143 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/v1/request/query/FileDownloadParams.java @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.api.model.v1.request.query; + +public interface FileDownloadParams { + + String getDefaultFileTypeName(); +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 787f9edc4..4e4a67f18 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -1,9 +1,11 @@ package org.breedinginsight.brapi.v2; +import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; +import io.micronaut.http.server.types.files.StreamedFile; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; @@ -16,15 +18,17 @@ import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.model.v1.validators.QueryValid; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; - import javax.inject.Inject; import javax.validation.Valid; -import java.util.HashMap; import java.util.List; import java.util.UUID; @@ -32,14 +36,15 @@ @Controller @Secured(SecurityRule.IS_AUTHENTICATED) public class ExperimentController { - private final BrAPITrialService experimentService; private final ExperimentQueryMapper experimentQueryMapper; + private final ProgramService programService; @Inject - public ExperimentController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper) { + public ExperimentController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService) { this.experimentService = experimentService; this.experimentQueryMapper = experimentQueryMapper; + this.programService = programService; } @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/trials{?queryParams*}") @@ -83,4 +88,29 @@ public HttpResponse> getExperimentById( return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); } } + + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}) + public HttpResponse datasetExport( + @PathVariable("programId") UUID programId, @PathVariable("experimentId") UUID experimentId, + @QueryValue @Valid ExperimentExportQuery queryParams) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); + DownloadFile datasetFile = experimentService.exportObservations(program, experimentId, queryParams); + HttpResponse response = HttpResponse + .ok(datasetFile.getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + datasetFile.getFileName()); + return response; + } catch (Exception e) { + log.info(e.getMessage(), e); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + + + } + + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java new file mode 100644 index 000000000..5f60f1008 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentExportQuery.java @@ -0,0 +1,24 @@ +package org.breedinginsight.brapi.v2.model.request.query; + +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import org.breedinginsight.api.model.v1.request.query.FilterRequest; +import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.jooq.tools.StringUtils; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Introspected +public class ExperimentExportQuery { + private FileType fileExtension; + private String dataset; + private String environments; + @NotNull + private boolean includeTimestamps; + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7bee585a3..4faa89dba 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,30 +1,72 @@ package org.breedinginsight.brapi.v2.services; - +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import org.brapi.client.v2.model.exceptions.ApiException; +import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; +import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.BrAPIConstants; +import org.breedinginsight.model.Column; +import org.breedinginsight.model.DownloadFile; +import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.utilities.Utilities; - import javax.inject.Inject; import javax.inject.Singleton; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.stream.Collectors; @Slf4j @Singleton public class BrAPITrialService { - + private final String referenceSource; private final BrAPITrialDAO trialDAO; + private final BrAPIObservationDAO observationDAO; + private final BrAPIListDAO listDAO; + private final BrAPIObservationVariableDAO obsVarDAO; + private final BrAPIStudyDAO studyDAO; + private final BrAPISeasonDAO seasonDAO; private final BrAPIObservationUnitDAO ouDAO; + private final BrAPIGermplasmDAO germplasmDAO; @Inject - public BrAPITrialService( BrAPITrialDAO trialDAO,BrAPIObservationUnitDAO ouDAO) { + public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, + BrAPITrialDAO trialDAO, + BrAPIObservationDAO observationDAO, + BrAPIListDAO listDAO, + BrAPIObservationVariableDAO obsVarDAO, + BrAPIStudyDAO studyDAO, + BrAPISeasonDAO seasonDAO, + BrAPIObservationUnitDAO ouDAO, + BrAPIGermplasmDAO germplasmDAO) { + + this.referenceSource = referenceSource; this.trialDAO = trialDAO; + this.observationDAO = observationDAO; + this.listDAO = listDAO; + this.obsVarDAO = obsVarDAO; + this.studyDAO = studyDAO; + this.seasonDAO = seasonDAO; this.ouDAO = ouDAO; + this.germplasmDAO = germplasmDAO; } public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { @@ -52,4 +94,305 @@ private long countGermplasm(UUID programId, String trialDbId) throws ApiExceptio List obUnits = ouDAO.getObservationUnitsForTrialDbId(programId, trialDbId); return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); } + + public DownloadFile exportObservations( + Program program, + UUID experimentId, + ExperimentExportQuery params) throws IOException, DoesNotExistException, ApiException { + StreamedFile downloadFile; + boolean isDataset = false; + List dataset = new ArrayList<>(); + List obsVars = new ArrayList<>(); + Map> rowByOUId = new HashMap<>(); + Map studyByDbId = new HashMap<>(); + List requestedEnvIds = StringUtils.isNotBlank(params.getEnvironments()) ? + new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); + FileType fileType = params.getFileExtension(); + + // get requested environments for the experiment + List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); + if (!requestedEnvIds.isEmpty()) { + expStudies = expStudies + .stream() + .filter(study -> requestedEnvIds.contains(getStudyId(study))) + .collect(Collectors.toList()); + } + expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); + + // get the OUs for the requested environments + List ous = new ArrayList<>(); + Map ouByOUDbId = new HashMap<>(); + try { + for (BrAPIStudy study: expStudies) { + List studyOUs = ouDAO.getObservationUnitsForStudyDbId(study.getStudyDbId(), program); + studyOUs.forEach(ou -> ouByOUDbId.put(ou.getObservationUnitDbId(), ou)); + ous.addAll(studyOUs); + } + } catch (ApiException err) { + log.error("Error fetching observation units for a study by its DbId" + + Utilities.generateApiExceptionLogMessage(err), err); + } + + // make columns present in all exports + List columns = ExperimentFileColumns.getOrderedColumns(); + + // add columns for requested dataset obsvars and timestamps + BrAPITrial experiment = getExperiment(program, experimentId); + if ((StringUtils.isBlank(params.getDataset()) || "observations".equalsIgnoreCase(params.getDataset())) && + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) != null) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + isDataset = true; + obsVars = getDatasetObsVars(obsDatasetId, program); + + // make additional columns in the export for each obs variable and obs variable timestamp + addObsVarColumns(columns, obsVars, params.isIncludeTimestamps(), program); + + } + + // make export rows from any observations + if (isDataset) { + dataset = observationDAO.getObservationsByTrialDbId(List.of(experiment.getTrialDbId()), program); + } + if (!requestedEnvIds.isEmpty()) { + dataset = filterDatasetByEnvironment(dataset, requestedEnvIds, studyByDbId); + } + + // update rowByOUId + addBrAPIObsToRecords( + dataset, + experiment, + program, + ouByOUDbId, + studyByDbId, + rowByOUId, + params.isIncludeTimestamps(), + obsVars + ); + + // make export rows for OUs without observations + if (rowByOUId.size() < ous.size()) { + for (BrAPIObservationUnit ou: ous) { + String ouId = getOUId(ou); + if (!rowByOUId.containsKey(ouId)) { + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId)); + } + } + } + + // write export data to requested file format + List> exportRows = new ArrayList<>(rowByOUId.values()); + if (fileType.equals(FileType.CSV)){ + downloadFile = CSVWriter.writeToDownload(columns, exportRows, fileType); + } else { + downloadFile = ExcelWriter.writeToDownload("Experiment Data", columns, exportRows, fileType); + } + + String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); + String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); + return new DownloadFile(fileName, downloadFile); + } + + private void addBrAPIObsToRecords( + List dataset, + BrAPITrial experiment, + Program program, + Map ouByOUDbId, + Map studyByDbId, + Map> rowByOUId, + boolean includeTimestamp, + List obsVars) throws ApiException, DoesNotExistException { + Map varByDbId = new HashMap<>(); + obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); + for (BrAPIObservation obs: dataset) { + + // get observation unit for observation + BrAPIObservationUnit ou = ouByOUDbId.get(obs.getObservationUnitDbId()); + String ouId = getOUId(ou); + + // get observation variable for BrAPI observation + BrAPIObservationVariable var = varByDbId.get(obs.getObservationVariableDbId()); + + // if there is a row with that ouId then just add the obs var data and timestamp to the row + if (rowByOUId.get(ouId) != null) { + addObsVarDataToRow(rowByOUId.get(ouId), obs, includeTimestamp, var, program); + } else { + + // otherwise make a new row + Map row = createExportRow(experiment, program, ou, studyByDbId); + addObsVarDataToRow(row, obs, includeTimestamp, var, program); + rowByOUId.put(ouId, row); + } + } + } + + private String getOUId(BrAPIObservationUnit ou) { + BrAPIExternalReference ouXref = Utilities.getExternalReference( + ou.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) + .orElseThrow(() -> new RuntimeException("observation unit id not found")); + return ouXref.getReferenceID(); + } + + private String getStudyId(BrAPIStudy study) { + BrAPIExternalReference studyXref = Utilities.getExternalReference( + study.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new RuntimeException("study id not found")); + return studyXref.getReferenceID(); + } + + private void addObsVarDataToRow( + Map row, + BrAPIObservation obs, + boolean includeTimestamp, + BrAPIObservationVariable var, + Program program) { + String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); + if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || + var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + row.put(varName, Double.parseDouble(obs.getValue())); + } else { + row.put(varName, obs.getValue()); + } + + if (includeTimestamp) { + String stamp = obs.getObservationTimeStamp() == null ? "" : obs.getObservationTimeStamp().toString(); + row.put(String.format("TS:%s",varName), stamp); + } + } + + public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { + List lists = listDAO.getListByTypeAndExternalRef( + BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (lists == null || lists.isEmpty()) { + throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); + } + String listDbId = lists.get(0).getListDbId(); + BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); + List obsVarNames = list.getResult().getData(); + return obsVarDAO.getVariableByName(obsVarNames, program.getId()); + } + + public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { + List experiments = trialDAO.getTrialsByExperimentIds(List.of(experimentId), program); + if (experiments.isEmpty()) { + throw new RuntimeException("A trial with given experiment id was not returned"); + } + + return experiments.get(0); + } + + private Map createExportRow( + BrAPITrial experiment, + Program program, + BrAPIObservationUnit ou, + Map studyByDbId) throws ApiException, DoesNotExistException { + HashMap row = new HashMap<>(); + + // get OU id, germplasm, and study + BrAPIExternalReference ouXref = Utilities.getExternalReference( + ou.getExternalReferences(), + String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) + .orElseThrow(() -> new RuntimeException("observation unit id not found")); + String ouId = ouXref.getReferenceID(); + BrAPIGermplasm germplasm = germplasmDAO.getGermplasmByDBID(ou.getGermplasmDbId(), program.getId()) + .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); + BrAPIStudy study = studyByDbId.get(ou.getStudyDbId()); + + // make export row from BrAPI objects + row.put(ExperimentObservation.Columns.GERMPLASM_NAME, Utilities.removeProgramKey(ou.getGermplasmName(), program.getKey(), germplasm.getAccessionNumber())); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, germplasm.getAccessionNumber()); + + // use only the capitalized first character of the entry type for test/check + BrAPIEntryTypeEnum entryType = ou.getObservationUnitPosition().getEntryType(); + String testCheck = entryType != null ? String.valueOf(Character.toUpperCase(entryType.toString().charAt(0))) : null; + row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); + row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); + row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); + row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); + row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); + row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); + row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + // get replicate number + Optional repLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(level -> BrAPIConstants.REPLICATE.getValue().equals(level.getLevelName())) + .findFirst(); + repLevel.ifPresent(brAPIObservationUnitLevelRelationship -> + row.put(ExperimentObservation.Columns.REP_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); + + //get block number + Optional blockLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(level -> BrAPIConstants.BLOCK.getValue().equals(level.getLevelName())) + .findFirst(); + blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> + row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); + if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && + ou.getObservationUnitPosition().getPositionCoordinateY() != null) { + row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); + row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); + } + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); + } else { + row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, null); + } + row.put(ExperimentObservation.Columns.OBS_UNIT_ID, ouId); + + return row; + } + + + + private void addObsVarColumns( + List columns, + List obsVars, + boolean includeTimestamps, + Program program) { + for (BrAPIObservationVariable var: obsVars) { + Column obsVarColumn = new Column(); + obsVarColumn.setDataType(Column.ColumnDataType.STRING); + if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || + var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + obsVarColumn.setDataType(Column.ColumnDataType.DOUBLE); + } + String varName = Utilities.removeProgramKey(var.getObservationVariableName(), program.getKey()); + obsVarColumn.setValue(varName); + columns.add(obsVarColumn); + if (includeTimestamps) { + columns.add(new Column(String.format("TS:%s",varName),Column.ColumnDataType.STRING)); + } + } + } + private String makeFileName(BrAPITrial experiment, Program program, String envName) { + // _Observation Dataset [-]__ + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + return String.format("%s_Observation Dataset [%s-%s]_%s_%s", + Utilities.removeProgramKey(experiment.getTrialName(), program.getKey()), + program.getKey(), + experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString(), + envName, + timestamp); + } + private List filterDatasetByEnvironment( + List dataset, + List envIds, + Map studyByDbId) { + return dataset + .stream() + .filter(obs -> envIds.contains(getStudyId(studyByDbId.get(obs.getStudyDbId())))) + .collect(Collectors.toList()); + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index ecaa4bff0..7083cae0e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -69,6 +69,22 @@ public List getObservationsByStudyName(List studyNames ); } + public List getObservationsByTrialDbId(List trialDbIds, Program program) throws ApiException { + if(trialDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest(); + observationSearchRequest.setProgramDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + observationSearchRequest.setTrialDbIds(new ArrayList<>(trialDbIds)); + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); + return brAPIDAOUtil.search( + api::searchObservationsPost, + (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), + observationSearchRequest + ); + } + public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { if(ouDbIds.isEmpty() || variableDbIds.isEmpty()) { return Collections.emptyList(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java index ab9de443c..0a9180936 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; +import java.util.stream.Collectors; @Singleton public class BrAPITrialDAO { @@ -162,6 +163,25 @@ public List getTrialsByDbIds(Collection trialDbIds, Program trialSearch ); } + + public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { + if(experimentIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); + trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + //trialSearch.trialDbIds(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); + trialSearch.externalReferenceIDs(experimentIds.stream().map(id -> id.toString()).collect(Collectors.toList())); + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); + return brAPIDAOUtil.search( + api::searchTrialsPost, + api::searchTrialsSearchResultsDbIdGet, + trialSearch + ); + } + } diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java new file mode 100644 index 000000000..5ddb65a76 --- /dev/null +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -0,0 +1,63 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.services.parsers.experiment; + +import com.sun.mail.imap.protocol.ENVELOPE; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.model.Column; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public enum ExperimentFileColumns { + + GERMPLASM_NAME(ExperimentObservation.Columns.GERMPLASM_NAME, Column.ColumnDataType.STRING), + GERMPLASM_GID(ExperimentObservation.Columns.GERMPLASM_GID, Column.ColumnDataType.STRING), + TEST_CHECK(ExperimentObservation.Columns.TEST_CHECK, Column.ColumnDataType.STRING), + EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), + EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), + EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING), + EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), + ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), + ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), + ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER), + EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), + REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), + BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), + ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING), + COLUMN(ExperimentObservation.Columns.COLUMN, Column.ColumnDataType.STRING), + TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING), + OBS_UNIT_ID(ExperimentObservation.Columns.OBS_UNIT_ID, Column.ColumnDataType.STRING); + + private final Column column; + + ExperimentFileColumns(String value, Column.ColumnDataType dataType) { + this.column = new Column(value, dataType); + } + + @Override + public String toString() { + return column.getValue(); + } + + public static List getOrderedColumns() { + return Arrays.stream(ExperimentFileColumns.values()) + .map(value -> value.column) + .collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java index dc77c0479..2a6194690 100644 --- a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java +++ b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java @@ -19,7 +19,9 @@ import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; import io.micronaut.http.exceptions.HttpStatusException; +import io.micronaut.http.server.types.files.StreamedFile; import org.apache.commons.lang3.tuple.Pair; import org.breedinginsight.api.model.v1.request.query.PaginationParams; import org.breedinginsight.api.model.v1.request.query.QueryParams; diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java new file mode 100644 index 000000000..fb5f79f92 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -0,0 +1,450 @@ +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.FileUtil; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import tech.tablesaw.api.ColumnType; +import tech.tablesaw.api.Row; +import tech.tablesaw.api.Table; +import javax.inject.Inject; +import java.io.*; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; +import static io.micronaut.http.HttpRequest.*; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ExperimentControllerIntegrationTest extends BrAPITest { + + private Program program; + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Get experiment import map + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Add traits to program + traits = createTraits(2); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + for (Trait trait : traits) { + Random random = new Random(); + + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. + // Double val1 = Math.random(); + + Float val1 = random.nextFloat(); + row1.put(trait.getObservationVariableName(), val1); + } + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, traits), + null, + true, + client, + program, + mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + envIds.add(importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString() + ); + } + + /* + Tests + - export empty dataset, single environment, csv format + - export empty dataset, single environment, xls format + - export empty dataset, single environment, xlsx format + - export populated dataset, single environment, csv format + - export populated dataset, single environment, xls format + - export populated dataset, single environment, xlsx format + - export empty dataset, multiple environment, csv format + - export empty dataset, multiple environment, xls format + - export empty dataset, multiple environment, xlsx format + - export populated dataset, multiple environment, csv format + - export populated dataset, multiple environment, xls format + - export populated dataset, multiple environment, xlsx format + */ + @ParameterizedTest + @CsvSource(value = {"true,false,CSV", "true,true,CSV", + "false,false,CSV", "false,true,CSV", + "true,false,XLS", "true,true,XLS", + "false,false,XLS", "false,true,XLS", + "true,false,XLSX", "true,true,XLSX", + "false,false,XLSX", "false,true,XLSX",}) + @SneakyThrows + void downloadDatasets(boolean includeTimestamps, boolean requestEnv, String extension) { + // Download test experiment + String envParam = "all=true"; + if (requestEnv) { + envParam = "environments=" + String.join(",", envIds); + } + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/export?includeTimestamps=%s&%s&fileExtension=%s", + program.getId().toString(), experimentId, includeTimestamps, envParam, extension)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + // Assert 200 response + assertEquals(HttpStatus.OK, response.getStatus()); + + // Assert file format fidelity + Map mediaTypeByExtension = new HashMap<>(); + mediaTypeByExtension.put("CSV", FileType.CSV.getMimeType()); + mediaTypeByExtension.put("XLS", FileType.XLS.getMimeType()); + mediaTypeByExtension.put("XLSX", FileType.XLSX.getMimeType()); + String downloadMediaType = response.getHeaders().getContentType().orElseThrow(Exception::new); + assertEquals(mediaTypeByExtension.get(extension), downloadMediaType); + + // Assert import/export fidelity and presence of observation units in export + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + checkDownloadTable(requestEnv, rows, download, includeTimestamps, extension); + } + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + if(traits != null) { + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); + } + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + private void checkDownloadTable( + boolean requestEnv, + List> importRows, + Table table, + boolean includeTimestamps, + String extension) { + // Filename is correct: _Observation Dataset [-]__ + List> requestedImportRows = importRows; + + // All columns included + Integer expectedColNumber = columns.size(); + if (includeTimestamps) { + expectedColNumber += traits.size(); + } + assertEquals(expectedColNumber, table.columnCount()); + + if (!requestEnv) { + + // All environments downloaded + importRows.stream() + .map(row -> row.get(ExperimentObservation.Columns.ENV).toString()) + .distinct() + .collect(Collectors.toList()) + .forEach(envName -> assertTrue(table.stringColumn("Env").contains(envName))); + + } else { + + // Only requested environment downloaded + requestedImportRows = importRows + .stream() + .filter(row -> row.get("Env").toString().equals("Env1")).collect(Collectors.toList()); + assertEquals(1, table.stringColumn("Env").countUnique()); + assertTrue(table.stringColumn("Env").contains("Env1")); + } + + List> matchingImportRows = new ArrayList<>(); + Optional> matchingImportRow; + + for (int rowNum = 0; rowNum < requestedImportRows.size(); rowNum++) { + Row downloadRow = table.row(rowNum); + + // sort order is not guaranteed to be th same as import, so find import row for corresponding export row + // by first matching environment and GID + matchingImportRow = requestedImportRows.stream().filter(row -> { + String gid = ExperimentObservation.Columns.GERMPLASM_GID; + String env = ExperimentObservation.Columns.ENV; + if (extension.equalsIgnoreCase(FileType.CSV.getName())) { + return Integer.parseInt(row.get(gid).toString()) == downloadRow.getInt(gid) && + row.get(env).equals(downloadRow.getString(env)); + } else { + return row.get(gid).equals(downloadRow.getString(gid)) && row.get(env).equals(downloadRow.getString(env)); + } + }).findAny(); + assertTrue(matchingImportRow.isPresent() && !matchingImportRow.get().isEmpty()); + + // then check the rest of the fields match + if (isMatchedRow(matchingImportRow.get(), downloadRow)) { + matchingImportRows.add(matchingImportRow.get()); + } + } + assertEquals(requestedImportRows.size(),matchingImportRows.size()); + + // Observation units populated + assertEquals(0, table.column("ObsUnitID").countMissing()); + assertEquals(requestedImportRows.size(), table.column("ObsUnitID").countUnique()); + } + + private boolean isMatchedRow(Map importRow, Row downloadRow) { + System.out.println("Validating row: " + downloadRow.getRowNumber()); + System.out.println("import columns: " + importRow.size()); + return importRow.entrySet().stream().filter(e -> { + String header = e.getKey(); + List importColumns = columns + .stream() + .filter(col -> header.equals(col.getValue())).collect(Collectors.toList()); + if (importColumns.size() != 1) { + return false; + } + Object expectedVal = null; + Object downloadedVal = null; + boolean doCompare = false; + + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.STRING)) { + expectedVal = e.getValue().toString(); + downloadedVal = downloadRow.getString(e.getKey()); + doCompare = true; + } + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.INTEGER)) { + expectedVal = Integer.parseInt(e.getValue().toString()); + downloadedVal = downloadRow.getInt(e.getKey()); + doCompare = true; + } + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.DOUBLE)) { + expectedVal = Double.parseDouble(e.getValue().toString()); + downloadedVal = downloadRow.getDouble(e.getKey()); + doCompare = true; + } + if (downloadRow.getColumnType(e.getKey()).equals(ColumnType.FLOAT)) { + expectedVal = e.getValue(); + downloadedVal = downloadRow.getFloat(e.getKey()); + doCompare = true; + } + System.out.println("Column: "+e.getKey()+", Expected: '"+ expectedVal +"', Received: '" + downloadedVal+"'"); + if(doCompare) { + assertEquals(expectedVal, downloadedVal); + return expectedVal.equals(downloadedVal); + } else { + return false; + } + }).count() == importRow.size(); + } +} diff --git a/src/test/resources/sql/ImportControllerIntegrationTest.sql b/src/test/resources/sql/ImportControllerIntegrationTest.sql index 42da3cd41..8a93b472e 100644 --- a/src/test/resources/sql/ImportControllerIntegrationTest.sql +++ b/src/test/resources/sql/ImportControllerIntegrationTest.sql @@ -30,6 +30,12 @@ INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,crea '[{"Name": "Chris-BB_germ_1", "Breeding Method": "BCR", "Source": "Wild"}, {"Name": "Chris-BB_germ_2", "Breeding Method": "BCR", "Source": "Wild"}, {"Name": "Chris-BB_germ_3", "Breeding Method": "BCR", "Source": "Wild"}, {"Name": "Chris-BB_germ_4", "Breeding Method": "BCR", "Source": "Wild"}]', false,user_id,user_id); +INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,created_by,updated_by) values + ('ExperimentsTemplateMap','ExperimentImport', + '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C )"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C )": true}]', + false,user_id,user_id); + END $$;