From f8fc57f61056d1e890636a4850aadb55345b0001 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 10 Mar 2022 13:32:21 -0500 Subject: [PATCH 1/8] [BI-1146] WIP --- .../controllers/ImportController.java | 3 + .../importer/model/imports/BrAPIImport.java | 2 - .../EnvironmentData.java | 33 ++ .../experimentObservation/ExperimentData.java | 45 +++ .../ExperimentImportService.java | 79 ++++ .../ExperimentObservation.java | 227 ++++++++++++ .../experimentObservation/FileData.java | 49 +++ .../ObservationUnitData.java | 26 ++ .../PhenotypingStudyWithDataImport.java | 61 ---- ...PhenotypingStudyWithDataImportService.java | 96 ----- .../processors/ExperimentProcessor.java | 342 ++++++++++++++++++ .../breedinginsight/utilities/Utilities.java | 29 ++ ...add_experiment_template_system_mapping.sql | 39 ++ ...entDataTypeControllerIntegrationTest.java} | 2 +- 14 files changed, 873 insertions(+), 160 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java create mode 100644 src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql rename src/test/java/org/breedinginsight/api/v1/controller/{EnvironmentTypeControllerIntegrationTest.java => EnvironmentDataTypeControllerIntegrationTest.java} (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 8475fdd17..fea1d075c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -199,6 +199,9 @@ public HttpResponse>> getSystemMappings(@Nu } else { result = fileImportService.getSystemMappingByName(actingUser, importName); } + log.info("..............."); + result.forEach((r) -> System.out.println(r.getId() + " ||| ")); + log.info("..............."); List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java index 9b960523e..ec2561a20 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java @@ -22,7 +22,6 @@ import java.util.List; public interface BrAPIImport { - default Germplasm getGermplasm() { return null; } default Trial getTrial() { return null; } default Location getLocation() { return null; } @@ -30,5 +29,4 @@ public interface BrAPIImport { default ObservationUnit getObservationUnit() { return null; } default List getObservations() { return null; } default ObservationVariable getObservationVariable() { return null; } - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java new file mode 100644 index 000000000..d67dd74e6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java @@ -0,0 +1,33 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +@Getter +@Setter +public class EnvironmentData { + private String env; + private String location; + private String year; + // TODO we don't want a getObservationUnitList() method + private List observationUnitDataList = new ArrayList<>(); + + public EnvironmentData(String env, String location, String year) { + this.env = env; + this.location = location; + this.year = year; + } + + public void addObservationUnitData(ObservationUnitData ou){ + this.observationUnitDataList.add(ou); + } + + public Collection observationUnitValues(){ + return this.observationUnitDataList; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java new file mode 100644 index 000000000..2be50f798 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java @@ -0,0 +1,45 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.Getter; +import lombok.Setter; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + + +public class ExperimentData { + @Getter + @Setter + private String title; + @Getter + @Setter + private String description; + + + private Map environments = new HashMap<>(); + + public ExperimentData(String title, String description) { + this.title = title; + this.description = description; + } + + public EnvironmentData retrieve_or_add_environmentData(String envName, String envLoc, String envYear){ + String key = envLoc + envYear; + EnvironmentData environmentData; + if( ! environments.containsKey(key) ){ + environmentData = new EnvironmentData(envName, envLoc, envYear); + environments.put(key, environmentData); + } + else{ + environmentData = this.environments.get(key); + } + return environmentData; + } + + public Collection environmentData_values(){ + return environments.values(); + } +} + + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java new file mode 100644 index 000000000..b8b556aaf --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -0,0 +1,79 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImport; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.services.processors.*; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public class ExperimentImportService extends BrAPIImportService { + + private final String IMPORT_TYPE_ID = "ExperimentImport"; + + private Provider experimentProcessorProvider; + private Provider processorManagerProvider; + + @Inject + public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + { + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + } + + @Override + public ExperimentObservation getImportClass() { + return new ExperimentObservation(); + } + + @Override + public String getImportTypeId() { + return IMPORT_TYPE_ID; + } + + @Override + public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + throws UnprocessableEntityException, ValidatorException, ApiException { + + ImportPreviewResponse response = null; + List processors = List.of(experimentProcessorProvider.get()); + try { + response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); + } catch (ValidatorException | ApiException e) { + log.error(e.getMessage()); + } + return response; + + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java new file mode 100644 index 000000000..d32bc6a18 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -0,0 +1,227 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapps.importer.model.config.*; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; + +@Getter +@Setter +@NoArgsConstructor +@ImportConfigMetadata(id="ExperimentImport", name="Experiment Import", + description = "This import is used to create Observation Unit and Experiment data") +public class ExperimentObservation implements BrAPIImport { + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="germplasmName", name="Germplasm Name", description = "Name of germplasm") + private String germplasmName; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="gid", name="Germplasm GID", description = "Unique germplasm identifier") + private String gid; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="test_or_check", name="Test or Check", description = "T test (T) and check (C) germplasm") + private String testOrCheck; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="exp_title", name="Experiment Title", description = "Title of experiment") + private String expTitle; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expDescription", name="Experiment Description", description = "Description of experiment") + private String expDescription; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expUnit", name="Experiment Unit", description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") + private String expUnit; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expType", name="Experiment Type", description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") + private String expType; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="env", name="Environment", description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") + private String env; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="envLocation", name="Environment Location", description = "Location of the environment") + private String envLocation; + + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id="envYear", name="Environment Year", description = "Year corresponding to the environment") + private String envYear; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expUnitId", name="Experiment Unit ID", description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") + private String expUnitId; + + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id="expReplicateNo", name="Experiment Replicate Number", description = "Sequential number of experimental replications") + private String expReplicateNo; + + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id="expBlockNo", name="Experiment Block Number", description = "Sequential number of blocks in an experimental design") + private String expBlockNo; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="row", name="Row", description = "Horizontal (y-axis) position in 2D Cartesian space.") + private String row; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="column", name="Column", description = "Vertical (x-axis) position in 2D Cartesian space.") + private String column; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="treatmentFactors", name="Exp Treatment Factor Name", description = "treatement factors in an experiment with applied variables, like fertilizer or water regimens.") + private String treatmentFactors; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="ObsUnitID", name="Observation Unit ID", description = "A database generated unique identifier for experimental observation units") + private String ObsUnitID; + + public BrAPITrial constructBrAPITrial(Program program, boolean commit) { + BrAPIProgram brapiProgram = program.getBrapiProgram(); + BrAPITrial trial = new BrAPITrial(); + if( commit ){ + trial.setTrialName( getTrialNameWithProgramKey( program )); + } + else{ + trial.setTrialName( getExpTitle() ); + } + trial.setTrialDescription(getExpDescription()); + trial.setActive(true); + trial.setProgramDbId(brapiProgram.getProgramDbId()); + trial.setProgramName(brapiProgram.getProgramName()); + + // TODO: Get to a constant + trial.putAdditionalInfoItem("defaultObservationLevel", getExpUnit()); + trial.putAdditionalInfoItem("experimentType", getExpType()); + return trial; + } + + public String getTrialNameWithProgramKey(Program program){ + return String.format("%s [%s]", getExpTitle(), program.getKey()); + } + + public BrAPILocation constructBrAPILocation() { + BrAPILocation location = new BrAPILocation(); + location.setLocationName(getEnvLocation()); + return location; + } + + public BrAPIStudy constructBrAPIStudy(Program program, Supplier nextVal, boolean commit) { + BrAPIStudy study = new BrAPIStudy(); + if ( commit ){ + study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), nextVal.get().toString())); + } + else { + study.setStudyName(getEnv()); + } + study.setActive(true); + study.setStudyType(getExpType()); + study.setLocationName(getEnvLocation()); + study.setTrialName(getExpTitle()); + study.setSeasons(List.of(getEnvYear())); + /* + TODO: Not used + BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); + design.setPUI(getExperimentalDesignPUI()); + study.setExperimentalDesign(design); + */ + + return study; + } + + public BrAPIObservationUnit constructBrAPIObservationUnit(Program program, Supplier nextVal, boolean commit) { + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + if( commit){ + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); + } + else { + observationUnit.setObservationUnitName(getExpUnitId()); + } + observationUnit.setStudyName(getEnv()); + + // TODO: Set the germplasm + //observationUnit.setGermplasmName(getGermplasm().getReferenceValue()); + + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + level.setLevelName(getExpUnit()); + position.setObservationLevel(level); + + // Exp Unit + List levelRelationships = new ArrayList<>(); + if( getExpReplicateNo() !=null ) { + BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); + repLvl.setLevelName("replicate"); + repLvl.setLevelCode(getExpReplicateNo()); + levelRelationships.add(repLvl); + } + + // Block number + if( getExpBlockNo() != null ) { + BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); + repLvl.setLevelName("block"); + repLvl.setLevelCode(getExpBlockNo()); + levelRelationships.add(repLvl); + } + position.setObservationLevelRelationships(levelRelationships); + + // Test or Check + if("C".equals(getTestOrCheck())){ + position.setEntryType(BrAPIEntryTypeEnum.CHECK); + } else { + position.setEntryType(BrAPIEntryTypeEnum.TEST); + } + + // X and Y coordinates + if (getRow() != null) { + position.setPositionCoordinateX(getRow()); + position.setPositionCoordinateXType(BrAPIPositionCoordinateTypeEnum.GRID_ROW); + } + if (getColumn() != null) { + position.setPositionCoordinateY(getColumn()); + position.setPositionCoordinateYType(BrAPIPositionCoordinateTypeEnum.GRID_ROW); + } + observationUnit.setObservationUnitPosition(position); + + // Treatment factors + if (getTreatmentFactors() != null) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(getTreatmentFactors()); + observationUnit.setTreatments(List.of(treatment)); + } + + return observationUnit; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java new file mode 100644 index 000000000..92ea67e4a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java @@ -0,0 +1,49 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.NoArgsConstructor; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +@NoArgsConstructor +public class FileData { + private Map expMap = new HashMap<>(); + private Map gids = new HashMap<>(); + + public ExperimentData retrieve_or_add_ExperimentData(String title, String description){ + String key = title + description; + ExperimentData experimentData = null; + if( ! expMap.containsKey(key) ){ + experimentData = new ExperimentData(title, description); + expMap.put(key,experimentData); + } + else { + experimentData = expMap.get(key); + } + return experimentData; + } + + public int experiments_size(){ + return expMap.size(); + } + + public ExperimentData get_ExperimentData(String key){ + return expMap.get(key); + } + + public Collection experimentData(){ + return expMap.values(); + } + + public void add_gid(String gid){ + if(!gids.containsKey(gid)){ + gids.put(gid,gid); + } + } + + public int gids_size(){ return gids.size(); } + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java new file mode 100644 index 000000000..d760c2fee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java @@ -0,0 +1,26 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.brapi.v2.model.pheno.BrAPIEntryTypeEnum; + +@Builder +@Getter +@Setter +public class ObservationUnitData { + private BrAPIEntryTypeEnum test_or_check = BrAPIEntryTypeEnum.TEST; + + private String germplasm_name = null; + private String gid = null; + private String exp_unit = null; + private String exp_unit_id = null; + private String exp_type = null; + private String exp_replicate_no = null; + private String exp_block_no = null; + private String row = null; + private String column = null; + private String treatment_factors = null; + private String id = null; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java deleted file mode 100644 index 9a639c7a1..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.model.imports.phenotyping; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.breedinginsight.brapps.importer.model.base.*; -import org.breedinginsight.brapps.importer.model.config.ImportConfigMetadata; -import org.breedinginsight.brapps.importer.model.config.ImportFieldType; -import org.breedinginsight.brapps.importer.model.config.ImportFieldTypeEnum; -import org.breedinginsight.brapps.importer.model.config.ImportMappingRequired; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; - -import java.util.List; - -@Getter -@Setter -@NoArgsConstructor -@ImportConfigMetadata(id="PhenotypingStudyWithDataImport", name="Phenotyping Study With Data", - description = "This import is used to create a phenotyping study including germplasm & observations.") -public class PhenotypingStudyWithDataImport implements BrAPIImport { - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Germplasm germplasm; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Trial trial; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Location location; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Study study; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private ObservationUnit observationUnit; - - @ImportFieldType(type= ImportFieldTypeEnum.LIST, clazz = Observation.class) - private List observations; - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java deleted file mode 100644 index 9ef424df7..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.model.imports.phenotyping; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.services.processors.*; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import javax.inject.Inject; -import javax.inject.Provider; -import javax.inject.Singleton; -import java.util.List; - -@Singleton -@Slf4j -public class PhenotypingStudyWithDataImportService extends BrAPIImportService { - - private final String IMPORT_TYPE_ID = "PhenotypingStudyWithDataImport"; - - private Provider germplasmProcessorProvider; - private Provider trialProcessorProvider; - private Provider locationProcessorProvider; - private Provider studyProcessorProvider; - private Provider observationUnitProcessorProvider; - private Provider observationProcessorProvider; - private Provider processorManagerProvider; - - @Inject - public PhenotypingStudyWithDataImportService(Provider germplasmProcessorProvider, - Provider trialProcessorProvider, - Provider locationProcessorProvider, - Provider studyProcessorProvider, - Provider observationUnitProcessorProvider, - Provider observationProcessorProvider, - Provider processorManagerProvider) - { - this.germplasmProcessorProvider = germplasmProcessorProvider; - this.trialProcessorProvider = trialProcessorProvider; - this.locationProcessorProvider = locationProcessorProvider; - this.studyProcessorProvider = studyProcessorProvider; - this.observationUnitProcessorProvider = observationUnitProcessorProvider; - this.observationProcessorProvider = observationProcessorProvider; - this.processorManagerProvider = processorManagerProvider; - } - - @Override - public PhenotypingStudyWithDataImport getImportClass() { - return new PhenotypingStudyWithDataImport(); - } - - @Override - public String getImportTypeId() { - return IMPORT_TYPE_ID; - } - - @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, ValidatorException, ApiException { - - ImportPreviewResponse response = null; - List processors = List.of(germplasmProcessorProvider.get(), - trialProcessorProvider.get(), - locationProcessorProvider.get(), - studyProcessorProvider.get(), - observationUnitProcessorProvider.get(), - observationProcessorProvider.get()); - - response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); - - return response; - } -} - diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java new file mode 100644 index 000000000..19a2387db --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -0,0 +1,342 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +//import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.*; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +@Slf4j +@Prototype +public class ExperimentProcessor implements Processor { + + private static final String NAME = "Experiment"; + + private FileData fileData = new FileData(); + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + private DSLContext dsl; + private BrAPITrialDAO brapiTrialDAO; + private BrAPILocationDAO brAPILocationDAO; + private BrAPIStudyDAO brAPIStudyDAO; + private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + + private List newTrialsList = new ArrayList<>(); + private List locationList = new ArrayList<>(); + private List studyList = new ArrayList<>(); + private List observationUnitList = new ArrayList<>(); + + private Map> trialByNameNoScope = new HashMap<>(); + private Map> locationByName = new HashMap<>(); + private Map> studyByNameNoScope = new HashMap<>(); + private Map> observationUnitByNameNoScope = new HashMap<>(); + + @Inject + public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.dsl = dsl; + this.brapiTrialDAO = brapiTrialDAO; + this.brAPILocationDAO = brAPILocationDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + + + } + + public void getExistingBrapiData(List importRows, Program program) { + + List experimentImportRows = importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + + // Trials + List uniqueTrialNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpTitle()) + .distinct() + .collect(Collectors.toList()); + List existingTrials; + + try { + existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + existingTrials.forEach(existingTrial -> { + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + + // Locations + List uniqueLocationNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnvLocation()) + .distinct() + .collect(Collectors.toList()); + + List existingLocations; + try { + existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + existingLocations.forEach(existingLocation -> { + locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + + // Studies + List uniqueStudyNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnv()) + .distinct() + .collect(Collectors.toList()); + + List existingStudies; + try { + existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies.forEach(existingStudy -> { + String studySequence = null; + if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { + studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); + } + studyByNameNoScope.put( + Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + + // Observation Unit + List uniqueObservationUnitNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpUnitId()) + .distinct() + .collect(Collectors.toList()); + + // check for existing observation units. Don't want to update existing, just create new. + // TODO: do we allow adding observations to existing studies? yes, but not updating + // ignore all data for observation units existing in system + + List existingObservationUnits; + + try { + existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); + existingObservationUnits.forEach(existingObservationUnit -> { + + // update mapped brapi import, does in process + observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + @Override + public Map process(List importRows, + Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + + // Validations + // GID for existing germplasm. Throw error if GID not found. + // Test or Check, bad letter throw an error. + + //TODO retrieve sequanceName from the program table. + String studySequenceName = ""; + Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); + String opsUnitSequenceName = ""; + Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + + // For each import row + for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + // Construct Trial + // Unique by trialName + + if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { + mappedImportRow.setTrial( trialByNameNoScope.get( importRow.getExpTitle() ) ); + } + else { + BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newTrial); + trialByNameNoScope.put(importRow.getExpTitle(), newImportObject); + newTrialsList.add(newTrial); + mappedImportRow.setTrial( newImportObject ); + } + + if( locationByName.containsKey(( importRow.getEnvLocation() ))){ + mappedImportRow.setLocation( locationByName.get( importRow.getEnvLocation() ) ); + } + else{ + BrAPILocation newLocation = importRow.constructBrAPILocation(); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + locationByName.put(importRow.getEnvLocation(), newImportObject); + mappedImportRow.setLocation( newImportObject ); + } + + if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { + mappedImportRow.setStudy( studyByNameNoScope.get( importRow.getEnv() ) ); + } + else{ + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit ); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newStudy); + studyByNameNoScope.put(importRow.getEnv(),newImportObject); + mappedImportRow.setStudy( newImportObject ); + } + + if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { + mappedImportRow.setObservationUnit( observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ); + } + else{ + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit ); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + observationUnitByNameNoScope.put(importRow.getExpUnitId(), new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit)); + mappedImportRow.setObservationUnit( newImportObject ); + } + + // Construct Observation Unit + // Manual test in breedbase to see observation level of any type supported + // Are we limiting to just plot and plant? + + // Construct Observations -- Done in another card + mappedBrAPIImport.put(i, mappedImportRow); + } + + // TODO need to add things to newExperimentList + // Construct our response object + ImportPreviewStatistics experimentStats = ImportPreviewStatistics.builder() + .newObjectCount(1) + .build(); + + + + //TODO What do we what mapped + return Map.of( + "Experiment", experimentStats + ); + } + + private void extracted() { + for (ExperimentData experimentData : this.fileData.experimentData()) { + // ?? Use BrAPITrial.constructBrAPITrial(BrAPIProgram brapiProgram) + BrAPITrial brAPITrial = new BrAPITrial(); + //Exp Title → Trial.trialName + brAPITrial.setTrialName(experimentData.getTitle()); + // Exp Description → Trial.trialDescription + brAPITrial.setTrialDescription(experimentData.getDescription()); + + //observationUnit → + for (EnvironmentData environmentData : experimentData.environmentData_values()) { + BrAPIStudy brAPIStudy = new BrAPIStudy(); + // TODO How dose this map? + // Exp Type → Study.studyType + // brAPIStudy.setStudyType( environment.); + + // Env → Study.studyName + brAPIStudy.setStudyName(environmentData.getEnv()); + + // TODO How does this map? brAPIStudy.setSeasons(seasons) wants a List for seasons + // Env Year → Study.seasons (This field must be numeric, and be a four digit year) + //brAPIStudy.setSeasons(); + + // Env Location → Study.locationDbId + brAPIStudy.setLocationDbId(environmentData.getLocation()); + // TODO lookup by name (getDbid) or else create a new Location + + for (ObservationUnitData observationUnitData : environmentData.getObservationUnitDataList()) { + BrAPIObservationUnitLevelRelationship brAPIobsUnitLevel = new BrAPIObservationUnitLevelRelationship(); + BrAPIObservationUnitPosition brAPIobsUnitPos = new BrAPIObservationUnitPosition(); + BrAPIObservationUnit brAPIobservationUnit = new BrAPIObservationUnit(); + brAPIobsUnitPos.setObservationLevel(brAPIobsUnitLevel); + brAPIobservationUnit.setObservationUnitPosition(brAPIobsUnitPos); + + //Test (T) or Check (C) → ObservationUnit.observationUnitPosition.entryType + brAPIobsUnitPos.setEntryType(observationUnitData.getTest_or_check()); + //Germplasm GID → ObservationUnit.germplasmDbId + //TODO This will require looking up the germplasm by external reference to get the correct DBID + brAPIobservationUnit.setGermplasmDbId(observationUnitData.getGid()); + + // Exp Unit → ObservationUnit.observationUnitPosition.observationLevel.levelName + brAPIobsUnitLevel.setLevelName(observationUnitData.getExp_unit()); + // Exp Unit → Trial.additionalInfo.defaultObservationLevel + brAPITrial.putAdditionalInfoItem("defaultObservationLevel", observationUnitData.getExp_unit()); + + // Exp Unit Id → ObservationUnit.observationUnitName + brAPIobservationUnit.setObservationUnitName(observationUnitData.getExp_unit_id()); + } + } + + } + } + + + @Override + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } + + @Override + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + + } + + private void updateDependencyValues(Map mappedBrAPIImport) { + // TODO + } + + @Override + public String getName() { + return NAME; + } + + + + + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 3078b1a07..ee992e764 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -62,4 +62,33 @@ public static String appendProgramKey(String original, String programKey, String return String.format("%s [%s]", original, programKey); } } + + /** + * Remove program key from a string. Returns a new value instead of altering original string. + * + * @param original + * @param programKey + * @param additionalKeyData + * @return + */ + public static String removeProgramKey(String original, String programKey, String additionalKeyData) { + if(StringUtils.isNotEmpty(additionalKeyData)) { + String keyValue = String.format(" [%s-%s]", programKey, additionalKeyData); + return original.replace(keyValue, ""); + } else { + String keyValue = String.format(" [%s]", programKey); + return original.replace(keyValue, ""); + } + } + + /** + * Remove program key from a string. Returns a new value instead of altering original string. + * + * @param original + * @param programKey + * @return + */ + public static String removeProgramKey(String original, String programKey) { + return removeProgramKey(original, programKey, null); + } } diff --git a/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql new file mode 100644 index 000000000..c7ff87677 --- /dev/null +++ b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql @@ -0,0 +1,39 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ +DECLARE +user_id UUID; +BEGIN + +user_id := (SELECT id FROM bi_user WHERE name = 'system'); + +insert into importer_mapping (id, name, import_type_id, mapping, file, draft, created_at, updated_at, created_by, updated_by) +values ( + uuid_generate_v4(), + 'GermplasmTemplateMap', + 'GermplasmImport', + '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent DBID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent DBID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', + '[{"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2"}]', + false, + '2021-11-18 20:55:50', + '2021-11-18 20:55:50', + user_id, + user_id + ); + +END $$; \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentTypeControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java similarity index 98% rename from src/test/java/org/breedinginsight/api/v1/controller/EnvironmentTypeControllerIntegrationTest.java rename to src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java index e01ef6648..fcc5b4039 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentTypeControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java @@ -41,7 +41,7 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class EnvironmentTypeControllerIntegrationTest extends DatabaseTest { +public class EnvironmentDataTypeControllerIntegrationTest extends DatabaseTest { private String validEnvironmentTypeId; private String validEnvironmentTypeName; From a4eb41929facc800f95748f76f8c4c9a456d7c71 Mon Sep 17 00:00:00 2001 From: Chris T Date: Wed, 20 Apr 2022 15:25:37 -0400 Subject: [PATCH 2/8] [BI-1146] experiment import: fix import template migration --- ....5.35__add_experiment_template_system_mapping.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql index c7ff87677..8e9c85147 100644 --- a/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql +++ b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql @@ -25,13 +25,13 @@ user_id := (SELECT id FROM bi_user WHERE name = 'system'); insert into importer_mapping (id, name, import_type_id, mapping, file, draft, created_at, updated_at, created_by, updated_by) values ( uuid_generate_v4(), - 'GermplasmTemplateMap', - 'GermplasmImport', - '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent DBID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent DBID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', - '[{"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2"}]', + 'ExperimentsTemplateMap', + 'ExperimentImport', + '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C )"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Exp Trt Factor Name"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "objectId": "ObsUnitID"}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Exp Trt Factor Name": "Jam application", "Test (T) or Check (C )": true}]', false, - '2021-11-18 20:55:50', - '2021-11-18 20:55:50', + '2022-04-20 20:55:50', + '2022-04-20 20:55:50', user_id, user_id ); From 8b34e4d07432e50ff386c40b1caef795b36608e1 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 26 Apr 2022 09:19:20 -0400 Subject: [PATCH 3/8] [BI-1146] support for summary stats --- .../controllers/ImportController.java | 3 - .../importer/model/imports/BrAPIImport.java | 2 + .../ExperimentImportService.java | 6 +- .../processors/ExperimentProcessor.java | 478 ++++++++++-------- 4 files changed, 284 insertions(+), 205 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index fea1d075c..8475fdd17 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -199,9 +199,6 @@ public HttpResponse>> getSystemMappings(@Nu } else { result = fileImportService.getSystemMappingByName(actingUser, importName); } - log.info("..............."); - result.forEach((r) -> System.out.println(r.getId() + " ||| ")); - log.info("..............."); List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java index ec2561a20..9b960523e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java @@ -22,6 +22,7 @@ import java.util.List; public interface BrAPIImport { + default Germplasm getGermplasm() { return null; } default Trial getTrial() { return null; } default Location getLocation() { return null; } @@ -29,4 +30,5 @@ public interface BrAPIImport { default ObservationUnit getObservationUnit() { return null; } default List getObservations() { return null; } default ObservationVariable getObservationVariable() { return null; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index b8b556aaf..8284f19db 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -67,11 +67,7 @@ public ImportPreviewResponse process(List brAPIImports, Table data, ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - try { - response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); - } catch (ValidatorException | ApiException e) { - log.error(e.getMessage()); - } + response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 19a2387db..73b4edfa0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -18,14 +18,18 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -//import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.*; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; @@ -66,255 +70,205 @@ public class ExperimentProcessor implements Processor { private BrAPILocationDAO brAPILocationDAO; private BrAPIStudyDAO brAPIStudyDAO; private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private List newTrialsList = new ArrayList<>(); - private List locationList = new ArrayList<>(); - private List studyList = new ArrayList<>(); - private List observationUnitList = new ArrayList<>(); - - private Map> trialByNameNoScope = new HashMap<>(); - private Map> locationByName = new HashMap<>(); - private Map> studyByNameNoScope = new HashMap<>(); - private Map> observationUnitByNameNoScope = new HashMap<>(); + //These are initially populated by the getExistingBrapiData() method, + // then updated by the process() method. + private Map> trialByNameNoScope = null; + private Map> locationByName = null; + private Map> studyByNameNoScope = null; + private Map> observationUnitByNameNoScope = null; + // existingGermplasmByGID is initially populated by getExistingBrapiData(), but not updated by the process() method + private Map> existingGermplasmByGID = null; @Inject - public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + public ExperimentProcessor(DSLContext dsl, + BrAPITrialDAO brapiTrialDAO, + BrAPILocationDAO brAPILocationDAO, + BrAPIStudyDAO brAPIStudyDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPIGermplasmDAO brAPIGermplasmDAO) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; this.brAPILocationDAO = brAPILocationDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - - + this.brAPIGermplasmDAO = brAPIGermplasmDAO; } + /** + * Initialize the Map objects with existing BrAPI Data. + * @param importRows + * @param program + */ public void getExistingBrapiData(List importRows, Program program) { List experimentImportRows = importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); - // Trials - List uniqueTrialNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getExpTitle()) - .distinct() - .collect(Collectors.toList()); - List existingTrials; - - try { - existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); - existingTrials.forEach(existingTrial -> { - trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - - // Locations - List uniqueLocationNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getEnvLocation()) - .distinct() - .collect(Collectors.toList()); - - List existingLocations; - try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); - existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - - // Studies - List uniqueStudyNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getEnv()) - .distinct() - .collect(Collectors.toList()); - - List existingStudies; - try { - existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); - existingStudies.forEach(existingStudy -> { - String studySequence = null; - if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { - studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); - } - studyByNameNoScope.put( - Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - - // Observation Unit - List uniqueObservationUnitNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getExpUnitId()) - .distinct() - .collect(Collectors.toList()); - - // check for existing observation units. Don't want to update existing, just create new. - // TODO: do we allow adding observations to existing studies? yes, but not updating - // ignore all data for observation units existing in system - - List existingObservationUnits; - - try { - existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); - existingObservationUnits.forEach(existingObservationUnit -> { - - // update mapped brapi import, does in process - observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } + this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); + this.locationByName = initialize_uniqueLocationNames ( program, experimentImportRows ); + this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); + this.observationUnitByNameNoScope = initialize_observationUnitByNameNoScope( program, experimentImportRows ); + this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); } + /** + * @param importRows - one element of the list for every row of the import file. + * @param mappedBrAPIImport - passed in by reference and modified withing this program (this will latter be passed to the front end for the preview) + * @param program + * @param user + * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * @return Map - used to display the summary statistics. + * @throws ValidatorException + */ @Override public Map process(List importRows, Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { - - // Validations - // GID for existing germplasm. Throw error if GID not found. - // Test or Check, bad letter throw an error. - //TODO retrieve sequanceName from the program table. String studySequenceName = ""; Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); String opsUnitSequenceName = ""; Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + ValidationErrors validationErrors = new ValidationErrors(); + + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet GIDCounter = new HashSet<>(); // set of unique GID's // For each import row for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - // Construct Trial - // Unique by trialName - if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { - mappedImportRow.setTrial( trialByNameNoScope.get( importRow.getExpTitle() ) ); - } - else { - BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newTrial); - trialByNameNoScope.put(importRow.getExpTitle(), newImportObject); - newTrialsList.add(newTrial); - mappedImportRow.setTrial( newImportObject ); - } + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); + addIfNotNull(GIDCounter, importRow.getGid()); - if( locationByName.containsKey(( importRow.getEnvLocation() ))){ - mappedImportRow.setLocation( locationByName.get( importRow.getEnvLocation() ) ); - } - else{ - BrAPILocation newLocation = importRow.constructBrAPILocation(); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newLocation); - locationByName.put(importRow.getEnvLocation(), newImportObject); - mappedImportRow.setLocation( newImportObject ); - } + //////////////////////////////////////////// + // Create and store PendingImportObject's // + //////////////////////////////////////////// + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow ); + mappedImportRow.setTrial( trialPIO ); + this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); - if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { - mappedImportRow.setStudy( studyByNameNoScope.get( importRow.getEnv() ) ); - } - else{ - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit ); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newStudy); - studyByNameNoScope.put(importRow.getEnv(),newImportObject); - mappedImportRow.setStudy( newImportObject ); - } + PendingImportObject locationPIO = createLocationPIO(importRow); + mappedImportRow.setLocation( locationPIO ); + this.locationByName.put( importRow.getEnvLocation(), locationPIO ); - if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { - mappedImportRow.setObservationUnit( observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ); - } - else{ - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit ); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); - observationUnitByNameNoScope.put(importRow.getExpUnitId(), new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit)); - mappedImportRow.setObservationUnit( newImportObject ); - } + PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); + mappedImportRow.setStudy( studyPIO ); + this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - // Construct Observation Unit - // Manual test in breedbase to see observation level of any type supported - // Are we limiting to just plot and plant? + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + mappedImportRow.setObservationUnit(obsUnitPIO); + this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + + PendingImportObject germplasmPIO = getGidPio(validationErrors, importRow, mappedImportRow); + mappedImportRow.setGermplasm( germplasmPIO ); + validateGerplam( validationErrors, i, germplasmPIO ); // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } + // End-of-loop + + if (validationErrors.hasErrors() ){ + throw new ValidatorException(validationErrors); + } - // TODO need to add things to newExperimentList // Construct our response object - ImportPreviewStatistics experimentStats = ImportPreviewStatistics.builder() - .newObjectCount(1) - .build(); + return getStatisticsMap(environmentNameCounter, obsUnitsIDCounter, GIDCounter); + } + private void addIfNotNull(HashSet set, String setValue) { + if( setValue!=null) { set.add(setValue); } + } + private Map getStatisticsMap(HashSet environmentNameSet, HashSet obsUnitsIDSet, HashSet GIDSet) { + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameSet.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDSet.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(GIDSet.size()) + .build(); - //TODO What do we what mapped return Map.of( - "Experiment", experimentStats + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats ); } - private void extracted() { - for (ExperimentData experimentData : this.fileData.experimentData()) { - // ?? Use BrAPITrial.constructBrAPITrial(BrAPIProgram brapiProgram) - BrAPITrial brAPITrial = new BrAPITrial(); - //Exp Title → Trial.trialName - brAPITrial.setTrialName(experimentData.getTitle()); - // Exp Description → Trial.trialDescription - brAPITrial.setTrialDescription(experimentData.getDescription()); - - //observationUnit → - for (EnvironmentData environmentData : experimentData.environmentData_values()) { - BrAPIStudy brAPIStudy = new BrAPIStudy(); - // TODO How dose this map? - // Exp Type → Study.studyType - // brAPIStudy.setStudyType( environment.); - - // Env → Study.studyName - brAPIStudy.setStudyName(environmentData.getEnv()); - - // TODO How does this map? brAPIStudy.setSeasons(seasons) wants a List for seasons - // Env Year → Study.seasons (This field must be numeric, and be a four digit year) - //brAPIStudy.setSeasons(); - - // Env Location → Study.locationDbId - brAPIStudy.setLocationDbId(environmentData.getLocation()); - // TODO lookup by name (getDbid) or else create a new Location - - for (ObservationUnitData observationUnitData : environmentData.getObservationUnitDataList()) { - BrAPIObservationUnitLevelRelationship brAPIobsUnitLevel = new BrAPIObservationUnitLevelRelationship(); - BrAPIObservationUnitPosition brAPIobsUnitPos = new BrAPIObservationUnitPosition(); - BrAPIObservationUnit brAPIobservationUnit = new BrAPIObservationUnit(); - brAPIobsUnitPos.setObservationLevel(brAPIobsUnitLevel); - brAPIobservationUnit.setObservationUnitPosition(brAPIobsUnitPos); - - //Test (T) or Check (C) → ObservationUnit.observationUnitPosition.entryType - brAPIobsUnitPos.setEntryType(observationUnitData.getTest_or_check()); - //Germplasm GID → ObservationUnit.germplasmDbId - //TODO This will require looking up the germplasm by external reference to get the correct DBID - brAPIobservationUnit.setGermplasmDbId(observationUnitData.getGid()); - - // Exp Unit → ObservationUnit.observationUnitPosition.observationLevel.levelName - brAPIobsUnitLevel.setLevelName(observationUnitData.getExp_unit()); - // Exp Unit → Trial.additionalInfo.defaultObservationLevel - brAPITrial.putAdditionalInfoItem("defaultObservationLevel", observationUnitData.getExp_unit()); - - // Exp Unit Id → ObservationUnit.observationUnitName - brAPIobservationUnit.setObservationUnitName(observationUnitData.getExp_unit_id()); - } - } + private void validateGerplam(ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { + if( germplasmPIO == null ) { + ValidationError ve = new ValidationError("GID", "Not an existing GID", HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + } + } + + private PendingImportObject getGidPio(ValidationErrors validationErrors, ExperimentObservation importRow, PendingImport mappedImportRow) { + if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ + return existingGermplasmByGID.get(importRow.getGid()); + } + else{ + return null; + } + } + + private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { + PendingImportObject pio = null; + if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { + pio = observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ; + } + else{ + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + } + return pio; + } + + private PendingImportObject createStudyPIO(Program program, boolean commit, Supplier studyNextVal, ExperimentObservation importRow ) { + PendingImportObject pio = null; + if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { + pio = studyByNameNoScope.get( importRow.getEnv() ) ; + } + else{ + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy); + } + return pio; + } + private PendingImportObject createLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio = null; + if( locationByName.containsKey(( importRow.getEnvLocation() ))){ + pio = locationByName.get( importRow.getEnvLocation() ); } + else{ + BrAPILocation newLocation = importRow.constructBrAPILocation(); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + } + return pio; } + private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow) { + PendingImportObject pio = null; + if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { + pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; + } + else { + BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial); + } + return pio; + } @Override public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { @@ -335,7 +289,137 @@ public String getName() { return NAME; } + //TODO should this method be moved to BrAPIExperimentService.java + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm: germplasmList) { + for (String accessionNumber: germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber().equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + + private Map> initialize_existingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + List uniqueGermplasmGID = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getGid()) + .distinct() + .collect(Collectors.toList()); + + List existingGermplasms; + try { + existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGID, program.getId()); + existingGermplasms.forEach(existingGermplasm -> { + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); + }); + return existingGermplasmByGID; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + private Map> initialize_observationUnitByNameNoScope(Program program, List experimentImportRows) { + + Map> observationUnitByNameNoScope = new HashMap<>(); + List uniqueObservationUnitNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpUnitId()) + .distinct() + .collect(Collectors.toList()); + + // check for existing observation units. Don't want to update existing, just create new. + // TODO: do we allow adding observations to existing studies? yes, but not updating + // ignore all data for observation units existing in system + + List existingObservationUnits; + + try { + existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); + existingObservationUnits.forEach(existingObservationUnit -> { + // update mapped brapi import, does in process + observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); + }); + return observationUnitByNameNoScope; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { + Map> studyByNameNoScope = new HashMap<>(); + List uniqueStudyNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnv()) + .distinct() + .collect(Collectors.toList()); + + List existingStudies; + try { + existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies.forEach(existingStudy -> { + String studySequence = null; + if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { + studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); + } + studyByNameNoScope.put( + Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); + }); + return studyByNameNoScope; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private Map> initialize_uniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); + List uniqueLocationNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnvLocation()) + .distinct() + .collect(Collectors.toList()); + + List existingLocations; + try { + existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + existingLocations.forEach(existingLocation -> { + locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + }); + return locationByName; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private Map> initialize_trialByNameNoScope(Program program, List experimentImportRows) { + Map> trialByNameNoScope = new HashMap<>(); + List uniqueTrialNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpTitle()) + .distinct() + .collect(Collectors.toList()); + List existingTrials; + try { + existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + existingTrials.forEach(existingTrial -> { + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); + }); + return trialByNameNoScope; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } From 4b4349b014e0418b66e3d9081ff8a82389554f8e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 29 Apr 2022 16:21:03 -0400 Subject: [PATCH 4/8] [BI-1146] refactor. added code for coditionally required cells --- .../ExperimentObservation.java | 2 +- .../importer/services/FileImportService.java | 2 +- .../importer/services/MappingManager.java | 8 +- .../processors/ExperimentProcessor.java | 137 ++++++++++++++---- 4 files changed, 116 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index d32bc6a18..7a2a27b2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -148,7 +148,7 @@ public BrAPIStudy constructBrAPIStudy(Program program, Supplier next study.setStudyType(getExpType()); study.setLocationName(getEnvLocation()); study.setTrialName(getExpTitle()); - study.setSeasons(List.of(getEnvYear())); + study.setSeasons( List.of( getEnvYear()==null ? "" : getEnvYear() ) ); /* TODO: Not used BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 00866bc1d..778519038 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -426,7 +426,7 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); } catch (ValidatorException e) { - log.error("Validation errors", e); + log.info("Validation errors", e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); progress.setMessage("Multiple Errors"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index 90c5af46d..b720349d9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -20,6 +20,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; @@ -46,6 +47,7 @@ import static org.breedinginsight.brapps.importer.model.config.ImportRelationType.DB_LOOKUP_CONSTANT_VALUE; +@Slf4j @Singleton public class MappingManager { @@ -110,8 +112,6 @@ private List map(ImportMapping importMapping, Table data, Map mappings, Table importFile, Integer rowIndex, Map userInput, Boolean process, ValidationErrors validationErrors) throws UnprocessableEntityException { - - Row focusRow = importFile.row(rowIndex); // Process this field ImportFieldType type = field.getAnnotation(ImportFieldType.class); @@ -125,7 +125,6 @@ private void mapField(Object parent, Field field, List mappings, T metadata = field.getAnnotation(ImportFieldMetadata.class) != null ? field.getAnnotation(ImportFieldMetadata.class) : (ImportFieldMetadata) type.clazz().getAnnotation(ImportFieldMetadata.class); } - ImportMappingRequired required = field.getAnnotation(ImportMappingRequired.class); // Check if it is a user input field @@ -136,7 +135,6 @@ private void mapField(Object parent, Field field, List mappings, T } return; } - List foundMappings = new ArrayList<>(); if (mappings != null) { foundMappings = mappings.stream() @@ -151,7 +149,6 @@ private void mapField(Object parent, Field field, List mappings, T } else if (required == null && foundMappings.size() == 0) { return; } - MappingField matchedMapping = foundMappings.get(0); if (type == null) { throw new InternalServerException("BrAPIObject is missing import type annotation"); @@ -310,6 +307,7 @@ private void mapField(Object parent, Field field, List mappings, T // Check non-null value if (required != null && value.isBlank()) { + //throw new UnprocessableEntityException(String.format(blankRequiredField, metadata.name())); ValidationError ve = getMissingRequiredErr(metadata.name()); validationErrors.addError(getRowNumber(rowIndex), ve); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 73b4edfa0..86a531630 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -141,41 +141,35 @@ public Map process(List importRows // For each import row for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); addIfNotNull(GIDCounter, importRow.getGid()); - //////////////////////////////////////////// - // Create and store PendingImportObject's // - //////////////////////////////////////////// - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow ); - mappedImportRow.setTrial( trialPIO ); - this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); - - PendingImportObject locationPIO = createLocationPIO(importRow); - mappedImportRow.setLocation( locationPIO ); - this.locationByName.put( importRow.getEnvLocation(), locationPIO ); - - PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); - mappedImportRow.setStudy( studyPIO ); - this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - mappedImportRow.setObservationUnit(obsUnitPIO); - this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); - - PendingImportObject germplasmPIO = getGidPio(validationErrors, importRow, mappedImportRow); - mappedImportRow.setGermplasm( germplasmPIO ); - validateGerplam( validationErrors, i, germplasmPIO ); + //////////////////////////////////////////////////// + // Create (or get) and store PendingImportObjects // + // into the PendingImport // + //////////////////////////////////////////////////// + PendingImport mappedImportRow = + getPendingImport( + mappedBrAPIImport, + program, + commit, + studyNextVal, + obsUnitNextVal, + i, + importRow); + + validateGerplam( validationErrors, i, mappedImportRow.getGermplasm() ); // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } // End-of-loop + validationErrors = validateConditionallyRequiredFields(importRows, validationErrors); + if (validationErrors.hasErrors() ){ throw new ValidatorException(validationErrors); } @@ -184,6 +178,92 @@ public Map process(List importRows return getStatisticsMap(environmentNameCounter, obsUnitsIDCounter, GIDCounter); } + private ValidationErrors validateConditionallyRequiredFields(List importRows, ValidationErrors validationErrors) { + for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + String experimentTitle = importRow.getExpTitle(); + ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); + boolean isExistingNew = (expState == ImportObjectState.NEW); + + if (isExistingNew) { + String errorMessage = "Field is blank when creating a new experiment"; + validateNotNullOrBlank( importRow.getGid(), + "GID", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpUnit(), + "Exp Unit", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpType(), + "Exp Type", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getEnv(), + "Env", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getEnvLocation(), + "Env Location", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getEnvYear(), + "Env Year", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpUnitId(), + "Exp Unit ID", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpReplicateNo(), + "Exp Replicate #", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpBlockNo(), + "Exp Block #", + errorMessage, validationErrors, i); + } + } + return validationErrors; + } + + private void validateNotNullOrBlank(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { + if ( value==null || value.isBlank()) { + addRowError( + columHeader, + errorMessage, + validationErrors, i + ) ; + } + } + + private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int i) { + ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + } + + private PendingImport getPendingImport( + Map mappedBrAPIImport, + Program program, + boolean commit, + Supplier studyNextVal, + Supplier obsUnitNextVal, + int i, + ExperimentObservation importRow) { + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); + mappedImportRow.setTrial( trialPIO ); + this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); + + PendingImportObject locationPIO = createLocationPIO(importRow); + mappedImportRow.setLocation( locationPIO ); + this.locationByName.put( importRow.getEnvLocation(), locationPIO ); + + PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); + mappedImportRow.setStudy( studyPIO ); + this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); + + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + mappedImportRow.setObservationUnit(obsUnitPIO); + this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + + PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); + mappedImportRow.setGermplasm( germplasmPIO ); + return mappedImportRow; + } + private void addIfNotNull(HashSet set, String setValue) { if( setValue!=null) { set.add(setValue); } } @@ -208,12 +288,12 @@ private Map getStatisticsMap(HashSet en private void validateGerplam(ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { if( germplasmPIO == null ) { - ValidationError ve = new ValidationError("GID", "Not an existing GID", HttpStatus.UNPROCESSABLE_ENTITY); + ValidationError ve = new ValidationError("GID", "A non-existing GID", HttpStatus.UNPROCESSABLE_ENTITY); validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. } } - private PendingImportObject getGidPio(ValidationErrors validationErrors, ExperimentObservation importRow, PendingImport mappedImportRow) { + private PendingImportObject getGidPio(ExperimentObservation importRow, PendingImport mappedImportRow) { if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ return existingGermplasmByGID.get(importRow.getGid()); } @@ -332,6 +412,7 @@ private Map> initialize_observ List uniqueObservationUnitNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getExpUnitId()) .distinct() + .filter(name -> null !=name) .collect(Collectors.toList()); // check for existing observation units. Don't want to update existing, just create new. @@ -358,6 +439,7 @@ private Map> initialize_studyByNameNoSco List uniqueStudyNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getEnv()) .distinct() + .filter(name -> null !=name) .collect(Collectors.toList()); List existingStudies; @@ -384,6 +466,7 @@ private Map> initialize_uniqueLocatio List uniqueLocationNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getEnvLocation()) .distinct() + .filter(name -> null !=name) .collect(Collectors.toList()); List existingLocations; @@ -403,9 +486,11 @@ private Map> initialize_trialByNameNoSco Map> trialByNameNoScope = new HashMap<>(); List uniqueTrialNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getExpTitle()) - .distinct() + .distinct(). + filter(name -> null !=name) .collect(Collectors.toList()); List existingTrials; +log.info(uniqueTrialNames.toString()); try { existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); From 6edb1946934e9b7ee3ca4809b6888952b75961e1 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 2 May 2022 10:44:53 -0400 Subject: [PATCH 5/8] [BI-1146] refactor. added getNewBrapiData() --- .../processors/ExperimentProcessor.java | 120 ++++++++---------- 1 file changed, 56 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 86a531630..e243d9762 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -73,12 +73,12 @@ public class ExperimentProcessor implements Processor { private final BrAPIGermplasmDAO brAPIGermplasmDAO; //These are initially populated by the getExistingBrapiData() method, - // then updated by the process() method. + // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; private Map> observationUnitByNameNoScope = null; - // existingGermplasmByGID is initially populated by getExistingBrapiData(), but not updated by the process() method + // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @Inject @@ -119,47 +119,35 @@ public void getExistingBrapiData(List importRows, Program program) * @param mappedBrAPIImport - passed in by reference and modified withing this program (this will latter be passed to the front end for the preview) * @param program * @param user - * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * false when used for preview only * @return Map - used to display the summary statistics. * @throws ValidatorException */ @Override - public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { - //TODO retrieve sequanceName from the program table. - String studySequenceName = ""; - Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); - String opsUnitSequenceName = ""; - Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + public Map process( + List importRows, + Map mappedBrAPIImport, + Program program, + User user, + boolean commit) throws ValidatorException { + ValidationErrors validationErrors = new ValidationErrors(); - // Data for stats. - HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names - HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet GIDCounter = new HashSet<>(); // set of unique GID's + // add "New" pending data to the BrapiData objects + getNewBrapiData(importRows, program, commit); // For each import row for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - // Collect date for stats. - addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); - addIfNotNull(GIDCounter, importRow.getGid()); - - //////////////////////////////////////////////////// - // Create (or get) and store PendingImportObjects // - // into the PendingImport // - //////////////////////////////////////////////////// - PendingImport mappedImportRow = - getPendingImport( - mappedBrAPIImport, - program, - commit, - studyNextVal, - obsUnitNextVal, - i, - importRow); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + mappedImportRow.setTrial( this.trialByNameNoScope.get( importRow.getExpTitle() ) ); + mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); + mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); + mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( importRow.getExpUnitId() ) ); + PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); + mappedImportRow.setGermplasm( germplasmPIO ); validateGerplam( validationErrors, i, mappedImportRow.getGermplasm() ); @@ -175,7 +163,31 @@ public Map process(List importRows } // Construct our response object - return getStatisticsMap(environmentNameCounter, obsUnitsIDCounter, GIDCounter); + return getStatisticsMap(importRows); + } + + private void getNewBrapiData(List importRows, Program program, boolean commit) { + + //TODO retrieve sequanceName from the program table. + String studySequenceName = ""; + Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); + String opsUnitSequenceName = ""; + Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); + this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); + + PendingImportObject locationPIO = createLocationPIO(importRow); + this.locationByName.put( importRow.getEnvLocation(), locationPIO ); + + PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); + this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); + + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + } } private ValidationErrors validateConditionallyRequiredFields(List importRows, ValidationErrors validationErrors) { @@ -234,41 +246,21 @@ private void addRowError(String field, String errorMessage, ValidationErrors val validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. } - private PendingImport getPendingImport( - Map mappedBrAPIImport, - Program program, - boolean commit, - Supplier studyNextVal, - Supplier obsUnitNextVal, - int i, - ExperimentObservation importRow) { - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); - mappedImportRow.setTrial( trialPIO ); - this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); - - PendingImportObject locationPIO = createLocationPIO(importRow); - mappedImportRow.setLocation( locationPIO ); - this.locationByName.put( importRow.getEnvLocation(), locationPIO ); - - PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); - mappedImportRow.setStudy( studyPIO ); - this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - mappedImportRow.setObservationUnit(obsUnitPIO); - this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); - - PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); - mappedImportRow.setGermplasm( germplasmPIO ); - return mappedImportRow; - } - private void addIfNotNull(HashSet set, String setValue) { if( setValue!=null) { set.add(setValue); } } - private Map getStatisticsMap(HashSet environmentNameSet, HashSet obsUnitsIDSet, HashSet GIDSet) { + private Map getStatisticsMap(List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet GIDCounter = new HashSet<>(); // set of unique GID's + for (int i = 0; i < importRows.size(); i++) { + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); + addIfNotNull(GIDCounter, importRow.getGid()); + } ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameSet.size()) .build(); From 8ec3dfe8eff89087a4c215753929647b2d835154 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 3 May 2022 08:40:05 -0400 Subject: [PATCH 6/8] [BI-1146] remove unneeded Classes, clean up code --- .../EnvironmentData.java | 33 ----- .../experimentObservation/ExperimentData.java | 45 ------- .../experimentObservation/FileData.java | 49 ------- .../ObservationUnitData.java | 26 ---- .../processors/ExperimentProcessor.java | 126 ++++++++++-------- 5 files changed, 72 insertions(+), 207 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java deleted file mode 100644 index d67dd74e6..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.Getter; -import lombok.Setter; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - - -@Getter -@Setter -public class EnvironmentData { - private String env; - private String location; - private String year; - // TODO we don't want a getObservationUnitList() method - private List observationUnitDataList = new ArrayList<>(); - - public EnvironmentData(String env, String location, String year) { - this.env = env; - this.location = location; - this.year = year; - } - - public void addObservationUnitData(ObservationUnitData ou){ - this.observationUnitDataList.add(ou); - } - - public Collection observationUnitValues(){ - return this.observationUnitDataList; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java deleted file mode 100644 index 2be50f798..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.Getter; -import lombok.Setter; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - - -public class ExperimentData { - @Getter - @Setter - private String title; - @Getter - @Setter - private String description; - - - private Map environments = new HashMap<>(); - - public ExperimentData(String title, String description) { - this.title = title; - this.description = description; - } - - public EnvironmentData retrieve_or_add_environmentData(String envName, String envLoc, String envYear){ - String key = envLoc + envYear; - EnvironmentData environmentData; - if( ! environments.containsKey(key) ){ - environmentData = new EnvironmentData(envName, envLoc, envYear); - environments.put(key, environmentData); - } - else{ - environmentData = this.environments.get(key); - } - return environmentData; - } - - public Collection environmentData_values(){ - return environments.values(); - } -} - - diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java deleted file mode 100644 index 92ea67e4a..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.NoArgsConstructor; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -@NoArgsConstructor -public class FileData { - private Map expMap = new HashMap<>(); - private Map gids = new HashMap<>(); - - public ExperimentData retrieve_or_add_ExperimentData(String title, String description){ - String key = title + description; - ExperimentData experimentData = null; - if( ! expMap.containsKey(key) ){ - experimentData = new ExperimentData(title, description); - expMap.put(key,experimentData); - } - else { - experimentData = expMap.get(key); - } - return experimentData; - } - - public int experiments_size(){ - return expMap.size(); - } - - public ExperimentData get_ExperimentData(String key){ - return expMap.get(key); - } - - public Collection experimentData(){ - return expMap.values(); - } - - public void add_gid(String gid){ - if(!gids.containsKey(gid)){ - gids.put(gid,gid); - } - } - - public int gids_size(){ return gids.size(); } - - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java deleted file mode 100644 index d760c2fee..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.AccessLevel; -import lombok.Builder; -import lombok.Getter; -import lombok.Setter; -import org.brapi.v2.model.pheno.BrAPIEntryTypeEnum; - -@Builder -@Getter -@Setter -public class ObservationUnitData { - private BrAPIEntryTypeEnum test_or_check = BrAPIEntryTypeEnum.TEST; - - private String germplasm_name = null; - private String gid = null; - private String exp_unit = null; - private String exp_unit_id = null; - private String exp_type = null; - private String exp_replicate_no = null; - private String exp_block_no = null; - private String row = null; - private String column = null; - private String treatment_factors = null; - private String id = null; -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e243d9762..16e7891c3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -21,6 +21,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; @@ -36,7 +37,6 @@ import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -60,8 +60,6 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private FileData fileData = new FileData(); - @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -72,7 +70,7 @@ public class ExperimentProcessor implements Processor { private BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; - //These are initially populated by the getExistingBrapiData() method, + //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; private Map> locationByName = null; @@ -116,7 +114,7 @@ public void getExistingBrapiData(List importRows, Program program) /** * @param importRows - one element of the list for every row of the import file. - * @param mappedBrAPIImport - passed in by reference and modified withing this program (this will latter be passed to the front end for the preview) + * @param mappedBrAPIImport - passed in by reference and modified within this program (this will latter be passed to the front end for the preview) * @param program * @param user * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) @@ -149,14 +147,15 @@ public Map process( PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); mappedImportRow.setGermplasm( germplasmPIO ); - validateGerplam( validationErrors, i, mappedImportRow.getGermplasm() ); - + if (! StringUtils.isBlank( importRow.getGid() )) { // if GID is blank, don't bother to check if it is valid. + validateGermplam(importRow,validationErrors, i, germplasmPIO); + } // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } // End-of-loop - validationErrors = validateConditionallyRequiredFields(importRows, validationErrors); + validationErrors = validateFields(importRows, validationErrors); if (validationErrors.hasErrors() ){ throw new ValidatorException(validationErrors); @@ -190,49 +189,63 @@ private void getNewBrapiData(List importRows, Program program, bool } } - private ValidationErrors validateConditionallyRequiredFields(List importRows, ValidationErrors validationErrors) { + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - String experimentTitle = importRow.getExpTitle(); - ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); - boolean isExistingNew = (expState == ImportObjectState.NEW); - - if (isExistingNew) { - String errorMessage = "Field is blank when creating a new experiment"; - validateNotNullOrBlank( importRow.getGid(), - "GID", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpUnit(), - "Exp Unit", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpType(), - "Exp Type", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getEnv(), - "Env", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getEnvLocation(), - "Env Location", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getEnvYear(), - "Env Year", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpUnitId(), - "Exp Unit ID", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpReplicateNo(), - "Exp Replicate #", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpBlockNo(), - "Exp Block #", - errorMessage, validationErrors, i); - } + + validateConditionallyRequired(validationErrors, i, importRow); } return validationErrors; } - private void validateNotNullOrBlank(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { - if ( value==null || value.isBlank()) { + private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { + String experimentTitle = importRow.getExpTitle(); + String obsUnitID = importRow.getObsUnitID(); + if( StringUtils.isBlank( obsUnitID )){ + validateRequiredCell( + experimentTitle, + "Exp Title", + "Field is blank", validationErrors, i + ); + } + + ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); + boolean isExperimentNew = (expState == ImportObjectState.NEW); + + if (isExperimentNew) { + String errorMessage = "Field is blank when creating a new experiment"; + validateRequiredCell( importRow.getGid(), + "GID", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpUnit(), + "Exp Unit", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpType(), + "Exp Type", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getEnv(), + "Env", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getEnvLocation(), + "Env Location", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getEnvYear(), + "Env Year", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpUnitId(), + "Exp Unit ID", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpReplicateNo(), + "Exp Replicate #", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpBlockNo(), + "Exp Block #", + errorMessage, validationErrors, i); + } + } + + private void validateRequiredCell(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { + if ( StringUtils.isBlank( value )) { addRowError( columHeader, errorMessage, @@ -254,21 +267,22 @@ private Map getStatisticsMap(List // Data for stats. HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet GIDCounter = new HashSet<>(); // set of unique GID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); - addIfNotNull(GIDCounter, importRow.getGid()); + addIfNotNull(gidCounter, importRow.getGid()); } ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameSet.size()) + .newObjectCount(environmentNameCounter.size()) .build(); ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDSet.size()) + .newObjectCount(obsUnitsIDCounter.size()) .build(); ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(GIDSet.size()) + .newObjectCount(gidCounter.size()) .build(); return Map.of( @@ -278,10 +292,14 @@ private Map getStatisticsMap(List ); } - private void validateGerplam(ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { - if( germplasmPIO == null ) { - ValidationError ve = new ValidationError("GID", "A non-existing GID", HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + private void validateGermplam(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { + // error if GID is not blank but GID dose not already exist + if( !StringUtils.isBlank( importRow.getGid()) && germplasmPIO == null ) { + addRowError( + "GID", + "A non-existing GID", + validationErrors, i + ); } } From 39295aeb88d30dc5fb9a57dcb0b9efbcd68c228e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 12:31:38 -0400 Subject: [PATCH 7/8] [BI-1146] added validation for unique observation Unit Id --- .../services/processors/ExperimentProcessor.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 16e7891c3..541160028 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -75,6 +75,8 @@ public class ExperimentProcessor implements Processor { private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; + // It is assumed that there are no pre-existing observation units for the given environment (so this will not be + // intitalized by getExistingBrapiData private Map> observationUnitByNameNoScope = null; // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -190,14 +192,26 @@ private void getNewBrapiData(List importRows, Program program, bool } private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { + HashSet uniqueStudyAndObsUnit = new HashSet<>(); for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - validateConditionallyRequired(validationErrors, i, importRow); + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, i, importRow); } return validationErrors; } + private void validateUniqueObsUnits(ValidationErrors validationErrors, HashSet uniqueStudyAndObsUnit, int i, ExperimentObservation importRow) { + String envIdPlusStudyId = importRow.getEnv() + importRow.getExpUnitId(); + if( uniqueStudyAndObsUnit.contains( envIdPlusStudyId )){ + String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); + this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); + } + else{ + uniqueStudyAndObsUnit.add( envIdPlusStudyId ); + } + } + private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { String experimentTitle = importRow.getExpTitle(); String obsUnitID = importRow.getObsUnitID(); From c02dc13d5c4e555002fe189222183a8d27b0c245 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 16:45:36 -0400 Subject: [PATCH 8/8] [BI-1146] Fixed bug. If spreadsheet cell is 'T', tableSaw assumes that it is a BOOLEAN --- .../brapps/importer/services/MappingManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index b720349d9..808784821 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -281,6 +281,9 @@ private void mapField(Object parent, Field field, List mappings, T } else if (columnType == ColumnType.INTEGER) { fileValue = String.valueOf(focusRow.getInt(matchedMapping.getValue().getFileFieldName())); } + else if (columnType == ColumnType.BOOLEAN) { + fileValue = String.valueOf(focusRow.getBoolean(matchedMapping.getValue().getFileFieldName())); + } else { fileValue = focusRow.getString(matchedMapping.getValue().getFileFieldName()); }