From e77ac44ae26b93a5703b56353754102feff68bc1 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 10 Mar 2022 13:32:21 -0500 Subject: [PATCH 01/24] [BI-1146] WIP --- .../controllers/ImportController.java | 3 + .../importer/model/imports/BrAPIImport.java | 2 - .../EnvironmentData.java | 33 ++ .../experimentObservation/ExperimentData.java | 45 +++ .../ExperimentImportService.java | 79 ++++ .../ExperimentObservation.java | 227 ++++++++++++ .../experimentObservation/FileData.java | 49 +++ .../ObservationUnitData.java | 26 ++ .../PhenotypingStudyWithDataImport.java | 61 ---- ...PhenotypingStudyWithDataImportService.java | 96 ----- .../processors/ExperimentProcessor.java | 342 ++++++++++++++++++ .../breedinginsight/utilities/Utilities.java | 29 ++ ...add_experiment_template_system_mapping.sql | 39 ++ ...entDataTypeControllerIntegrationTest.java} | 2 +- 14 files changed, 873 insertions(+), 160 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java create mode 100644 src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql rename src/test/java/org/breedinginsight/api/v1/controller/{EnvironmentTypeControllerIntegrationTest.java => EnvironmentDataTypeControllerIntegrationTest.java} (98%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 8475fdd17..fea1d075c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -199,6 +199,9 @@ public HttpResponse>> getSystemMappings(@Nu } else { result = fileImportService.getSystemMappingByName(actingUser, importName); } + log.info("..............."); + result.forEach((r) -> System.out.println(r.getId() + " ||| ")); + log.info("..............."); List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java index 9b960523e..ec2561a20 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java @@ -22,7 +22,6 @@ import java.util.List; public interface BrAPIImport { - default Germplasm getGermplasm() { return null; } default Trial getTrial() { return null; } default Location getLocation() { return null; } @@ -30,5 +29,4 @@ public interface BrAPIImport { default ObservationUnit getObservationUnit() { return null; } default List getObservations() { return null; } default ObservationVariable getObservationVariable() { return null; } - } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java new file mode 100644 index 000000000..d67dd74e6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java @@ -0,0 +1,33 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +@Getter +@Setter +public class EnvironmentData { + private String env; + private String location; + private String year; + // TODO we don't want a getObservationUnitList() method + private List observationUnitDataList = new ArrayList<>(); + + public EnvironmentData(String env, String location, String year) { + this.env = env; + this.location = location; + this.year = year; + } + + public void addObservationUnitData(ObservationUnitData ou){ + this.observationUnitDataList.add(ou); + } + + public Collection observationUnitValues(){ + return this.observationUnitDataList; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java new file mode 100644 index 000000000..2be50f798 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java @@ -0,0 +1,45 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.Getter; +import lombok.Setter; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + + +public class ExperimentData { + @Getter + @Setter + private String title; + @Getter + @Setter + private String description; + + + private Map environments = new HashMap<>(); + + public ExperimentData(String title, String description) { + this.title = title; + this.description = description; + } + + public EnvironmentData retrieve_or_add_environmentData(String envName, String envLoc, String envYear){ + String key = envLoc + envYear; + EnvironmentData environmentData; + if( ! environments.containsKey(key) ){ + environmentData = new EnvironmentData(envName, envLoc, envYear); + environments.put(key, environmentData); + } + else{ + environmentData = this.environments.get(key); + } + return environmentData; + } + + public Collection environmentData_values(){ + return environments.values(); + } +} + + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java new file mode 100644 index 000000000..b8b556aaf --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -0,0 +1,79 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImport; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.services.processors.*; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public class ExperimentImportService extends BrAPIImportService { + + private final String IMPORT_TYPE_ID = "ExperimentImport"; + + private Provider experimentProcessorProvider; + private Provider processorManagerProvider; + + @Inject + public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + { + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + } + + @Override + public ExperimentObservation getImportClass() { + return new ExperimentObservation(); + } + + @Override + public String getImportTypeId() { + return IMPORT_TYPE_ID; + } + + @Override + public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + throws UnprocessableEntityException, ValidatorException, ApiException { + + ImportPreviewResponse response = null; + List processors = List.of(experimentProcessorProvider.get()); + try { + response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); + } catch (ValidatorException | ApiException e) { + log.error(e.getMessage()); + } + return response; + + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java new file mode 100644 index 000000000..d32bc6a18 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -0,0 +1,227 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapps.importer.model.config.*; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; + +@Getter +@Setter +@NoArgsConstructor +@ImportConfigMetadata(id="ExperimentImport", name="Experiment Import", + description = "This import is used to create Observation Unit and Experiment data") +public class ExperimentObservation implements BrAPIImport { + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="germplasmName", name="Germplasm Name", description = "Name of germplasm") + private String germplasmName; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="gid", name="Germplasm GID", description = "Unique germplasm identifier") + private String gid; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="test_or_check", name="Test or Check", description = "T test (T) and check (C) germplasm") + private String testOrCheck; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="exp_title", name="Experiment Title", description = "Title of experiment") + private String expTitle; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expDescription", name="Experiment Description", description = "Description of experiment") + private String expDescription; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expUnit", name="Experiment Unit", description = "experiment unit (Examples: plots, plant, tanks, hives, etc.)") + private String expUnit; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expType", name="Experiment Type", description = "Description of experimental type (Examples: Performance trial, crossing block, seed orchard, etc)") + private String expType; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="env", name="Environment", description = "Free-text unique identifier for environment within the experiment. Common examples include: 1,2,3…n and/or a concationation of environment location and year") + private String env; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="envLocation", name="Environment Location", description = "Location of the environment") + private String envLocation; + + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id="envYear", name="Environment Year", description = "Year corresponding to the environment") + private String envYear; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="expUnitId", name="Experiment Unit ID", description = "Human-readable alphanumeric identifier for experimental units unique within environment. Examples, like plot number, are often a numeric sequence.") + private String expUnitId; + + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id="expReplicateNo", name="Experiment Replicate Number", description = "Sequential number of experimental replications") + private String expReplicateNo; + + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + @ImportFieldMetadata(id="expBlockNo", name="Experiment Block Number", description = "Sequential number of blocks in an experimental design") + private String expBlockNo; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="row", name="Row", description = "Horizontal (y-axis) position in 2D Cartesian space.") + private String row; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="column", name="Column", description = "Vertical (x-axis) position in 2D Cartesian space.") + private String column; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="treatmentFactors", name="Exp Treatment Factor Name", description = "treatement factors in an experiment with applied variables, like fertilizer or water regimens.") + private String treatmentFactors; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id="ObsUnitID", name="Observation Unit ID", description = "A database generated unique identifier for experimental observation units") + private String ObsUnitID; + + public BrAPITrial constructBrAPITrial(Program program, boolean commit) { + BrAPIProgram brapiProgram = program.getBrapiProgram(); + BrAPITrial trial = new BrAPITrial(); + if( commit ){ + trial.setTrialName( getTrialNameWithProgramKey( program )); + } + else{ + trial.setTrialName( getExpTitle() ); + } + trial.setTrialDescription(getExpDescription()); + trial.setActive(true); + trial.setProgramDbId(brapiProgram.getProgramDbId()); + trial.setProgramName(brapiProgram.getProgramName()); + + // TODO: Get to a constant + trial.putAdditionalInfoItem("defaultObservationLevel", getExpUnit()); + trial.putAdditionalInfoItem("experimentType", getExpType()); + return trial; + } + + public String getTrialNameWithProgramKey(Program program){ + return String.format("%s [%s]", getExpTitle(), program.getKey()); + } + + public BrAPILocation constructBrAPILocation() { + BrAPILocation location = new BrAPILocation(); + location.setLocationName(getEnvLocation()); + return location; + } + + public BrAPIStudy constructBrAPIStudy(Program program, Supplier nextVal, boolean commit) { + BrAPIStudy study = new BrAPIStudy(); + if ( commit ){ + study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), nextVal.get().toString())); + } + else { + study.setStudyName(getEnv()); + } + study.setActive(true); + study.setStudyType(getExpType()); + study.setLocationName(getEnvLocation()); + study.setTrialName(getExpTitle()); + study.setSeasons(List.of(getEnvYear())); + /* + TODO: Not used + BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); + design.setPUI(getExperimentalDesignPUI()); + study.setExperimentalDesign(design); + */ + + return study; + } + + public BrAPIObservationUnit constructBrAPIObservationUnit(Program program, Supplier nextVal, boolean commit) { + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + if( commit){ + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); + } + else { + observationUnit.setObservationUnitName(getExpUnitId()); + } + observationUnit.setStudyName(getEnv()); + + // TODO: Set the germplasm + //observationUnit.setGermplasmName(getGermplasm().getReferenceValue()); + + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + level.setLevelName(getExpUnit()); + position.setObservationLevel(level); + + // Exp Unit + List levelRelationships = new ArrayList<>(); + if( getExpReplicateNo() !=null ) { + BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); + repLvl.setLevelName("replicate"); + repLvl.setLevelCode(getExpReplicateNo()); + levelRelationships.add(repLvl); + } + + // Block number + if( getExpBlockNo() != null ) { + BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); + repLvl.setLevelName("block"); + repLvl.setLevelCode(getExpBlockNo()); + levelRelationships.add(repLvl); + } + position.setObservationLevelRelationships(levelRelationships); + + // Test or Check + if("C".equals(getTestOrCheck())){ + position.setEntryType(BrAPIEntryTypeEnum.CHECK); + } else { + position.setEntryType(BrAPIEntryTypeEnum.TEST); + } + + // X and Y coordinates + if (getRow() != null) { + position.setPositionCoordinateX(getRow()); + position.setPositionCoordinateXType(BrAPIPositionCoordinateTypeEnum.GRID_ROW); + } + if (getColumn() != null) { + position.setPositionCoordinateY(getColumn()); + position.setPositionCoordinateYType(BrAPIPositionCoordinateTypeEnum.GRID_ROW); + } + observationUnit.setObservationUnitPosition(position); + + // Treatment factors + if (getTreatmentFactors() != null) { + BrAPIObservationTreatment treatment = new BrAPIObservationTreatment(); + treatment.setFactor(getTreatmentFactors()); + observationUnit.setTreatments(List.of(treatment)); + } + + return observationUnit; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java new file mode 100644 index 000000000..92ea67e4a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java @@ -0,0 +1,49 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.NoArgsConstructor; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +@NoArgsConstructor +public class FileData { + private Map expMap = new HashMap<>(); + private Map gids = new HashMap<>(); + + public ExperimentData retrieve_or_add_ExperimentData(String title, String description){ + String key = title + description; + ExperimentData experimentData = null; + if( ! expMap.containsKey(key) ){ + experimentData = new ExperimentData(title, description); + expMap.put(key,experimentData); + } + else { + experimentData = expMap.get(key); + } + return experimentData; + } + + public int experiments_size(){ + return expMap.size(); + } + + public ExperimentData get_ExperimentData(String key){ + return expMap.get(key); + } + + public Collection experimentData(){ + return expMap.values(); + } + + public void add_gid(String gid){ + if(!gids.containsKey(gid)){ + gids.put(gid,gid); + } + } + + public int gids_size(){ return gids.size(); } + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java new file mode 100644 index 000000000..d760c2fee --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java @@ -0,0 +1,26 @@ +package org.breedinginsight.brapps.importer.model.imports.experimentObservation; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.brapi.v2.model.pheno.BrAPIEntryTypeEnum; + +@Builder +@Getter +@Setter +public class ObservationUnitData { + private BrAPIEntryTypeEnum test_or_check = BrAPIEntryTypeEnum.TEST; + + private String germplasm_name = null; + private String gid = null; + private String exp_unit = null; + private String exp_unit_id = null; + private String exp_type = null; + private String exp_replicate_no = null; + private String exp_block_no = null; + private String row = null; + private String column = null; + private String treatment_factors = null; + private String id = null; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java deleted file mode 100644 index 9a639c7a1..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImport.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.model.imports.phenotyping; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.breedinginsight.brapps.importer.model.base.*; -import org.breedinginsight.brapps.importer.model.config.ImportConfigMetadata; -import org.breedinginsight.brapps.importer.model.config.ImportFieldType; -import org.breedinginsight.brapps.importer.model.config.ImportFieldTypeEnum; -import org.breedinginsight.brapps.importer.model.config.ImportMappingRequired; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; - -import java.util.List; - -@Getter -@Setter -@NoArgsConstructor -@ImportConfigMetadata(id="PhenotypingStudyWithDataImport", name="Phenotyping Study With Data", - description = "This import is used to create a phenotyping study including germplasm & observations.") -public class PhenotypingStudyWithDataImport implements BrAPIImport { - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Germplasm germplasm; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Trial trial; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Location location; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private Study study; - - @ImportFieldType(type = ImportFieldTypeEnum.OBJECT) - @ImportMappingRequired - private ObservationUnit observationUnit; - - @ImportFieldType(type= ImportFieldTypeEnum.LIST, clazz = Observation.class) - private List observations; - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java deleted file mode 100644 index 9ef424df7..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/phenotyping/PhenotypingStudyWithDataImportService.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.model.imports.phenotyping; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.services.processors.*; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import javax.inject.Inject; -import javax.inject.Provider; -import javax.inject.Singleton; -import java.util.List; - -@Singleton -@Slf4j -public class PhenotypingStudyWithDataImportService extends BrAPIImportService { - - private final String IMPORT_TYPE_ID = "PhenotypingStudyWithDataImport"; - - private Provider germplasmProcessorProvider; - private Provider trialProcessorProvider; - private Provider locationProcessorProvider; - private Provider studyProcessorProvider; - private Provider observationUnitProcessorProvider; - private Provider observationProcessorProvider; - private Provider processorManagerProvider; - - @Inject - public PhenotypingStudyWithDataImportService(Provider germplasmProcessorProvider, - Provider trialProcessorProvider, - Provider locationProcessorProvider, - Provider studyProcessorProvider, - Provider observationUnitProcessorProvider, - Provider observationProcessorProvider, - Provider processorManagerProvider) - { - this.germplasmProcessorProvider = germplasmProcessorProvider; - this.trialProcessorProvider = trialProcessorProvider; - this.locationProcessorProvider = locationProcessorProvider; - this.studyProcessorProvider = studyProcessorProvider; - this.observationUnitProcessorProvider = observationUnitProcessorProvider; - this.observationProcessorProvider = observationProcessorProvider; - this.processorManagerProvider = processorManagerProvider; - } - - @Override - public PhenotypingStudyWithDataImport getImportClass() { - return new PhenotypingStudyWithDataImport(); - } - - @Override - public String getImportTypeId() { - return IMPORT_TYPE_ID; - } - - @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, ValidatorException, ApiException { - - ImportPreviewResponse response = null; - List processors = List.of(germplasmProcessorProvider.get(), - trialProcessorProvider.get(), - locationProcessorProvider.get(), - studyProcessorProvider.get(), - observationUnitProcessorProvider.get(), - observationProcessorProvider.get()); - - response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); - - return response; - } -} - diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java new file mode 100644 index 000000000..19a2387db --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -0,0 +1,342 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapps.importer.services.processors; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +//import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.*; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +@Slf4j +@Prototype +public class ExperimentProcessor implements Processor { + + private static final String NAME = "Experiment"; + + private FileData fileData = new FileData(); + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + private DSLContext dsl; + private BrAPITrialDAO brapiTrialDAO; + private BrAPILocationDAO brAPILocationDAO; + private BrAPIStudyDAO brAPIStudyDAO; + private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + + private List newTrialsList = new ArrayList<>(); + private List locationList = new ArrayList<>(); + private List studyList = new ArrayList<>(); + private List observationUnitList = new ArrayList<>(); + + private Map> trialByNameNoScope = new HashMap<>(); + private Map> locationByName = new HashMap<>(); + private Map> studyByNameNoScope = new HashMap<>(); + private Map> observationUnitByNameNoScope = new HashMap<>(); + + @Inject + public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.dsl = dsl; + this.brapiTrialDAO = brapiTrialDAO; + this.brAPILocationDAO = brAPILocationDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + + + } + + public void getExistingBrapiData(List importRows, Program program) { + + List experimentImportRows = importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + + // Trials + List uniqueTrialNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpTitle()) + .distinct() + .collect(Collectors.toList()); + List existingTrials; + + try { + existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + existingTrials.forEach(existingTrial -> { + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + + // Locations + List uniqueLocationNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnvLocation()) + .distinct() + .collect(Collectors.toList()); + + List existingLocations; + try { + existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + existingLocations.forEach(existingLocation -> { + locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + + // Studies + List uniqueStudyNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnv()) + .distinct() + .collect(Collectors.toList()); + + List existingStudies; + try { + existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies.forEach(existingStudy -> { + String studySequence = null; + if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { + studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); + } + studyByNameNoScope.put( + Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + + // Observation Unit + List uniqueObservationUnitNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpUnitId()) + .distinct() + .collect(Collectors.toList()); + + // check for existing observation units. Don't want to update existing, just create new. + // TODO: do we allow adding observations to existing studies? yes, but not updating + // ignore all data for observation units existing in system + + List existingObservationUnits; + + try { + existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); + existingObservationUnits.forEach(existingObservationUnit -> { + + // update mapped brapi import, does in process + observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); + }); + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + @Override + public Map process(List importRows, + Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { + + // Validations + // GID for existing germplasm. Throw error if GID not found. + // Test or Check, bad letter throw an error. + + //TODO retrieve sequanceName from the program table. + String studySequenceName = ""; + Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); + String opsUnitSequenceName = ""; + Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + + // For each import row + for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + // Construct Trial + // Unique by trialName + + if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { + mappedImportRow.setTrial( trialByNameNoScope.get( importRow.getExpTitle() ) ); + } + else { + BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newTrial); + trialByNameNoScope.put(importRow.getExpTitle(), newImportObject); + newTrialsList.add(newTrial); + mappedImportRow.setTrial( newImportObject ); + } + + if( locationByName.containsKey(( importRow.getEnvLocation() ))){ + mappedImportRow.setLocation( locationByName.get( importRow.getEnvLocation() ) ); + } + else{ + BrAPILocation newLocation = importRow.constructBrAPILocation(); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + locationByName.put(importRow.getEnvLocation(), newImportObject); + mappedImportRow.setLocation( newImportObject ); + } + + if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { + mappedImportRow.setStudy( studyByNameNoScope.get( importRow.getEnv() ) ); + } + else{ + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit ); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newStudy); + studyByNameNoScope.put(importRow.getEnv(),newImportObject); + mappedImportRow.setStudy( newImportObject ); + } + + if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { + mappedImportRow.setObservationUnit( observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ); + } + else{ + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit ); + PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + observationUnitByNameNoScope.put(importRow.getExpUnitId(), new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit)); + mappedImportRow.setObservationUnit( newImportObject ); + } + + // Construct Observation Unit + // Manual test in breedbase to see observation level of any type supported + // Are we limiting to just plot and plant? + + // Construct Observations -- Done in another card + mappedBrAPIImport.put(i, mappedImportRow); + } + + // TODO need to add things to newExperimentList + // Construct our response object + ImportPreviewStatistics experimentStats = ImportPreviewStatistics.builder() + .newObjectCount(1) + .build(); + + + + //TODO What do we what mapped + return Map.of( + "Experiment", experimentStats + ); + } + + private void extracted() { + for (ExperimentData experimentData : this.fileData.experimentData()) { + // ?? Use BrAPITrial.constructBrAPITrial(BrAPIProgram brapiProgram) + BrAPITrial brAPITrial = new BrAPITrial(); + //Exp Title → Trial.trialName + brAPITrial.setTrialName(experimentData.getTitle()); + // Exp Description → Trial.trialDescription + brAPITrial.setTrialDescription(experimentData.getDescription()); + + //observationUnit → + for (EnvironmentData environmentData : experimentData.environmentData_values()) { + BrAPIStudy brAPIStudy = new BrAPIStudy(); + // TODO How dose this map? + // Exp Type → Study.studyType + // brAPIStudy.setStudyType( environment.); + + // Env → Study.studyName + brAPIStudy.setStudyName(environmentData.getEnv()); + + // TODO How does this map? brAPIStudy.setSeasons(seasons) wants a List for seasons + // Env Year → Study.seasons (This field must be numeric, and be a four digit year) + //brAPIStudy.setSeasons(); + + // Env Location → Study.locationDbId + brAPIStudy.setLocationDbId(environmentData.getLocation()); + // TODO lookup by name (getDbid) or else create a new Location + + for (ObservationUnitData observationUnitData : environmentData.getObservationUnitDataList()) { + BrAPIObservationUnitLevelRelationship brAPIobsUnitLevel = new BrAPIObservationUnitLevelRelationship(); + BrAPIObservationUnitPosition brAPIobsUnitPos = new BrAPIObservationUnitPosition(); + BrAPIObservationUnit brAPIobservationUnit = new BrAPIObservationUnit(); + brAPIobsUnitPos.setObservationLevel(brAPIobsUnitLevel); + brAPIobservationUnit.setObservationUnitPosition(brAPIobsUnitPos); + + //Test (T) or Check (C) → ObservationUnit.observationUnitPosition.entryType + brAPIobsUnitPos.setEntryType(observationUnitData.getTest_or_check()); + //Germplasm GID → ObservationUnit.germplasmDbId + //TODO This will require looking up the germplasm by external reference to get the correct DBID + brAPIobservationUnit.setGermplasmDbId(observationUnitData.getGid()); + + // Exp Unit → ObservationUnit.observationUnitPosition.observationLevel.levelName + brAPIobsUnitLevel.setLevelName(observationUnitData.getExp_unit()); + // Exp Unit → Trial.additionalInfo.defaultObservationLevel + brAPITrial.putAdditionalInfoItem("defaultObservationLevel", observationUnitData.getExp_unit()); + + // Exp Unit Id → ObservationUnit.observationUnitName + brAPIobservationUnit.setObservationUnitName(observationUnitData.getExp_unit_id()); + } + } + + } + } + + + @Override + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } + + @Override + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + + } + + private void updateDependencyValues(Map mappedBrAPIImport) { + // TODO + } + + @Override + public String getName() { + return NAME; + } + + + + + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 3078b1a07..ee992e764 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -62,4 +62,33 @@ public static String appendProgramKey(String original, String programKey, String return String.format("%s [%s]", original, programKey); } } + + /** + * Remove program key from a string. Returns a new value instead of altering original string. + * + * @param original + * @param programKey + * @param additionalKeyData + * @return + */ + public static String removeProgramKey(String original, String programKey, String additionalKeyData) { + if(StringUtils.isNotEmpty(additionalKeyData)) { + String keyValue = String.format(" [%s-%s]", programKey, additionalKeyData); + return original.replace(keyValue, ""); + } else { + String keyValue = String.format(" [%s]", programKey); + return original.replace(keyValue, ""); + } + } + + /** + * Remove program key from a string. Returns a new value instead of altering original string. + * + * @param original + * @param programKey + * @return + */ + public static String removeProgramKey(String original, String programKey) { + return removeProgramKey(original, programKey, null); + } } diff --git a/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql new file mode 100644 index 000000000..c7ff87677 --- /dev/null +++ b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql @@ -0,0 +1,39 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ +DECLARE +user_id UUID; +BEGIN + +user_id := (SELECT id FROM bi_user WHERE name = 'system'); + +insert into importer_mapping (id, name, import_type_id, mapping, file, draft, created_at, updated_at, created_by, updated_by) +values ( + uuid_generate_v4(), + 'GermplasmTemplateMap', + 'GermplasmImport', + '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent DBID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent DBID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', + '[{"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2"}]', + false, + '2021-11-18 20:55:50', + '2021-11-18 20:55:50', + user_id, + user_id + ); + +END $$; \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentTypeControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java similarity index 98% rename from src/test/java/org/breedinginsight/api/v1/controller/EnvironmentTypeControllerIntegrationTest.java rename to src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java index a98eff8a1..85465468b 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentTypeControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/EnvironmentDataTypeControllerIntegrationTest.java @@ -41,7 +41,7 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class EnvironmentTypeControllerIntegrationTest extends DatabaseTest { +public class EnvironmentDataTypeControllerIntegrationTest extends DatabaseTest { private String validEnvironmentTypeId; private String validEnvironmentTypeName; From 8245f0b691973344655ce59d15971b950b0c059b Mon Sep 17 00:00:00 2001 From: Chris T Date: Wed, 20 Apr 2022 15:25:37 -0400 Subject: [PATCH 02/24] [BI-1146] experiment import: fix import template migration --- ....5.35__add_experiment_template_system_mapping.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql index c7ff87677..8e9c85147 100644 --- a/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql +++ b/src/main/resources/db/migration/V0.5.35__add_experiment_template_system_mapping.sql @@ -25,13 +25,13 @@ user_id := (SELECT id FROM bi_user WHERE name = 'system'); insert into importer_mapping (id, name, import_type_id, mapping, file, draft, created_at, updated_at, created_by, updated_by) values ( uuid_generate_v4(), - 'GermplasmTemplateMap', - 'GermplasmImport', - '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent DBID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent DBID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', - '[{"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent DBID": "", "Female Parent DBID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent DBID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": ""}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent DBID": "", "Female Parent DBID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2"}]', + 'ExperimentsTemplateMap', + 'ExperimentImport', + '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C )"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Exp Trt Factor Name"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "objectId": "ObsUnitID"}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Exp Trt Factor Name": "Jam application", "Test (T) or Check (C )": true}]', false, - '2021-11-18 20:55:50', - '2021-11-18 20:55:50', + '2022-04-20 20:55:50', + '2022-04-20 20:55:50', user_id, user_id ); From f345aabe1b3b0e03ffb6bc0cad9f288697f52704 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 26 Apr 2022 09:19:20 -0400 Subject: [PATCH 03/24] [BI-1146] support for summary stats --- .../controllers/ImportController.java | 3 - .../importer/model/imports/BrAPIImport.java | 2 + .../ExperimentImportService.java | 6 +- .../processors/ExperimentProcessor.java | 478 ++++++++++-------- 4 files changed, 284 insertions(+), 205 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index fea1d075c..8475fdd17 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -199,9 +199,6 @@ public HttpResponse>> getSystemMappings(@Nu } else { result = fileImportService.getSystemMappingByName(actingUser, importName); } - log.info("..............."); - result.forEach((r) -> System.out.println(r.getId() + " ||| ")); - log.info("..............."); List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java index ec2561a20..9b960523e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImport.java @@ -22,6 +22,7 @@ import java.util.List; public interface BrAPIImport { + default Germplasm getGermplasm() { return null; } default Trial getTrial() { return null; } default Location getLocation() { return null; } @@ -29,4 +30,5 @@ public interface BrAPIImport { default ObservationUnit getObservationUnit() { return null; } default List getObservations() { return null; } default ObservationVariable getObservationVariable() { return null; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index b8b556aaf..8284f19db 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -67,11 +67,7 @@ public ImportPreviewResponse process(List brAPIImports, Table data, ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - try { - response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); - } catch (ValidatorException | ApiException e) { - log.error(e.getMessage()); - } + response = processorManagerProvider.get().process(brAPIImports, processors, program, upload, user, commit); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 19a2387db..73b4edfa0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -18,14 +18,18 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -//import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.*; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; @@ -66,255 +70,205 @@ public class ExperimentProcessor implements Processor { private BrAPILocationDAO brAPILocationDAO; private BrAPIStudyDAO brAPIStudyDAO; private BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPIGermplasmDAO brAPIGermplasmDAO; - private List newTrialsList = new ArrayList<>(); - private List locationList = new ArrayList<>(); - private List studyList = new ArrayList<>(); - private List observationUnitList = new ArrayList<>(); - - private Map> trialByNameNoScope = new HashMap<>(); - private Map> locationByName = new HashMap<>(); - private Map> studyByNameNoScope = new HashMap<>(); - private Map> observationUnitByNameNoScope = new HashMap<>(); + //These are initially populated by the getExistingBrapiData() method, + // then updated by the process() method. + private Map> trialByNameNoScope = null; + private Map> locationByName = null; + private Map> studyByNameNoScope = null; + private Map> observationUnitByNameNoScope = null; + // existingGermplasmByGID is initially populated by getExistingBrapiData(), but not updated by the process() method + private Map> existingGermplasmByGID = null; @Inject - public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + public ExperimentProcessor(DSLContext dsl, + BrAPITrialDAO brapiTrialDAO, + BrAPILocationDAO brAPILocationDAO, + BrAPIStudyDAO brAPIStudyDAO, + BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPIGermplasmDAO brAPIGermplasmDAO) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; this.brAPILocationDAO = brAPILocationDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - - + this.brAPIGermplasmDAO = brAPIGermplasmDAO; } + /** + * Initialize the Map objects with existing BrAPI Data. + * @param importRows + * @param program + */ public void getExistingBrapiData(List importRows, Program program) { List experimentImportRows = importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); - // Trials - List uniqueTrialNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getExpTitle()) - .distinct() - .collect(Collectors.toList()); - List existingTrials; - - try { - existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); - existingTrials.forEach(existingTrial -> { - trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - - // Locations - List uniqueLocationNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getEnvLocation()) - .distinct() - .collect(Collectors.toList()); - - List existingLocations; - try { - existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); - existingLocations.forEach(existingLocation -> { - locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - - // Studies - List uniqueStudyNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getEnv()) - .distinct() - .collect(Collectors.toList()); - - List existingStudies; - try { - existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); - existingStudies.forEach(existingStudy -> { - String studySequence = null; - if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { - studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); - } - studyByNameNoScope.put( - Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - - // Observation Unit - List uniqueObservationUnitNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getExpUnitId()) - .distinct() - .collect(Collectors.toList()); - - // check for existing observation units. Don't want to update existing, just create new. - // TODO: do we allow adding observations to existing studies? yes, but not updating - // ignore all data for observation units existing in system - - List existingObservationUnits; - - try { - existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); - existingObservationUnits.forEach(existingObservationUnit -> { - - // update mapped brapi import, does in process - observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } + this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); + this.locationByName = initialize_uniqueLocationNames ( program, experimentImportRows ); + this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); + this.observationUnitByNameNoScope = initialize_observationUnitByNameNoScope( program, experimentImportRows ); + this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); } + /** + * @param importRows - one element of the list for every row of the import file. + * @param mappedBrAPIImport - passed in by reference and modified withing this program (this will latter be passed to the front end for the preview) + * @param program + * @param user + * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * @return Map - used to display the summary statistics. + * @throws ValidatorException + */ @Override public Map process(List importRows, Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { - - // Validations - // GID for existing germplasm. Throw error if GID not found. - // Test or Check, bad letter throw an error. - //TODO retrieve sequanceName from the program table. String studySequenceName = ""; Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); String opsUnitSequenceName = ""; Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + ValidationErrors validationErrors = new ValidationErrors(); + + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet GIDCounter = new HashSet<>(); // set of unique GID's // For each import row for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - // Construct Trial - // Unique by trialName - if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { - mappedImportRow.setTrial( trialByNameNoScope.get( importRow.getExpTitle() ) ); - } - else { - BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newTrial); - trialByNameNoScope.put(importRow.getExpTitle(), newImportObject); - newTrialsList.add(newTrial); - mappedImportRow.setTrial( newImportObject ); - } + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); + addIfNotNull(GIDCounter, importRow.getGid()); - if( locationByName.containsKey(( importRow.getEnvLocation() ))){ - mappedImportRow.setLocation( locationByName.get( importRow.getEnvLocation() ) ); - } - else{ - BrAPILocation newLocation = importRow.constructBrAPILocation(); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newLocation); - locationByName.put(importRow.getEnvLocation(), newImportObject); - mappedImportRow.setLocation( newImportObject ); - } + //////////////////////////////////////////// + // Create and store PendingImportObject's // + //////////////////////////////////////////// + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow ); + mappedImportRow.setTrial( trialPIO ); + this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); - if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { - mappedImportRow.setStudy( studyByNameNoScope.get( importRow.getEnv() ) ); - } - else{ - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit ); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newStudy); - studyByNameNoScope.put(importRow.getEnv(),newImportObject); - mappedImportRow.setStudy( newImportObject ); - } + PendingImportObject locationPIO = createLocationPIO(importRow); + mappedImportRow.setLocation( locationPIO ); + this.locationByName.put( importRow.getEnvLocation(), locationPIO ); - if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { - mappedImportRow.setObservationUnit( observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ); - } - else{ - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit ); - PendingImportObject newImportObject = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); - observationUnitByNameNoScope.put(importRow.getExpUnitId(), new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit)); - mappedImportRow.setObservationUnit( newImportObject ); - } + PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); + mappedImportRow.setStudy( studyPIO ); + this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - // Construct Observation Unit - // Manual test in breedbase to see observation level of any type supported - // Are we limiting to just plot and plant? + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + mappedImportRow.setObservationUnit(obsUnitPIO); + this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + + PendingImportObject germplasmPIO = getGidPio(validationErrors, importRow, mappedImportRow); + mappedImportRow.setGermplasm( germplasmPIO ); + validateGerplam( validationErrors, i, germplasmPIO ); // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } + // End-of-loop + + if (validationErrors.hasErrors() ){ + throw new ValidatorException(validationErrors); + } - // TODO need to add things to newExperimentList // Construct our response object - ImportPreviewStatistics experimentStats = ImportPreviewStatistics.builder() - .newObjectCount(1) - .build(); + return getStatisticsMap(environmentNameCounter, obsUnitsIDCounter, GIDCounter); + } + private void addIfNotNull(HashSet set, String setValue) { + if( setValue!=null) { set.add(setValue); } + } + private Map getStatisticsMap(HashSet environmentNameSet, HashSet obsUnitsIDSet, HashSet GIDSet) { + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameSet.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDSet.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(GIDSet.size()) + .build(); - //TODO What do we what mapped return Map.of( - "Experiment", experimentStats + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats ); } - private void extracted() { - for (ExperimentData experimentData : this.fileData.experimentData()) { - // ?? Use BrAPITrial.constructBrAPITrial(BrAPIProgram brapiProgram) - BrAPITrial brAPITrial = new BrAPITrial(); - //Exp Title → Trial.trialName - brAPITrial.setTrialName(experimentData.getTitle()); - // Exp Description → Trial.trialDescription - brAPITrial.setTrialDescription(experimentData.getDescription()); - - //observationUnit → - for (EnvironmentData environmentData : experimentData.environmentData_values()) { - BrAPIStudy brAPIStudy = new BrAPIStudy(); - // TODO How dose this map? - // Exp Type → Study.studyType - // brAPIStudy.setStudyType( environment.); - - // Env → Study.studyName - brAPIStudy.setStudyName(environmentData.getEnv()); - - // TODO How does this map? brAPIStudy.setSeasons(seasons) wants a List for seasons - // Env Year → Study.seasons (This field must be numeric, and be a four digit year) - //brAPIStudy.setSeasons(); - - // Env Location → Study.locationDbId - brAPIStudy.setLocationDbId(environmentData.getLocation()); - // TODO lookup by name (getDbid) or else create a new Location - - for (ObservationUnitData observationUnitData : environmentData.getObservationUnitDataList()) { - BrAPIObservationUnitLevelRelationship brAPIobsUnitLevel = new BrAPIObservationUnitLevelRelationship(); - BrAPIObservationUnitPosition brAPIobsUnitPos = new BrAPIObservationUnitPosition(); - BrAPIObservationUnit brAPIobservationUnit = new BrAPIObservationUnit(); - brAPIobsUnitPos.setObservationLevel(brAPIobsUnitLevel); - brAPIobservationUnit.setObservationUnitPosition(brAPIobsUnitPos); - - //Test (T) or Check (C) → ObservationUnit.observationUnitPosition.entryType - brAPIobsUnitPos.setEntryType(observationUnitData.getTest_or_check()); - //Germplasm GID → ObservationUnit.germplasmDbId - //TODO This will require looking up the germplasm by external reference to get the correct DBID - brAPIobservationUnit.setGermplasmDbId(observationUnitData.getGid()); - - // Exp Unit → ObservationUnit.observationUnitPosition.observationLevel.levelName - brAPIobsUnitLevel.setLevelName(observationUnitData.getExp_unit()); - // Exp Unit → Trial.additionalInfo.defaultObservationLevel - brAPITrial.putAdditionalInfoItem("defaultObservationLevel", observationUnitData.getExp_unit()); - - // Exp Unit Id → ObservationUnit.observationUnitName - brAPIobservationUnit.setObservationUnitName(observationUnitData.getExp_unit_id()); - } - } + private void validateGerplam(ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { + if( germplasmPIO == null ) { + ValidationError ve = new ValidationError("GID", "Not an existing GID", HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + } + } + + private PendingImportObject getGidPio(ValidationErrors validationErrors, ExperimentObservation importRow, PendingImport mappedImportRow) { + if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ + return existingGermplasmByGID.get(importRow.getGid()); + } + else{ + return null; + } + } + + private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { + PendingImportObject pio = null; + if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { + pio = observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ; + } + else{ + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); + } + return pio; + } + + private PendingImportObject createStudyPIO(Program program, boolean commit, Supplier studyNextVal, ExperimentObservation importRow ) { + PendingImportObject pio = null; + if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { + pio = studyByNameNoScope.get( importRow.getEnv() ) ; + } + else{ + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy); + } + return pio; + } + private PendingImportObject createLocationPIO(ExperimentObservation importRow) { + PendingImportObject pio = null; + if( locationByName.containsKey(( importRow.getEnvLocation() ))){ + pio = locationByName.get( importRow.getEnvLocation() ); } + else{ + BrAPILocation newLocation = importRow.constructBrAPILocation(); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation); + } + return pio; } + private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow) { + PendingImportObject pio = null; + if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { + pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; + } + else { + BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); + pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial); + } + return pio; + } @Override public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { @@ -335,7 +289,137 @@ public String getName() { return NAME; } + //TODO should this method be moved to BrAPIExperimentService.java + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm: germplasmList) { + for (String accessionNumber: germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber().equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + + private Map> initialize_existingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + List uniqueGermplasmGID = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getGid()) + .distinct() + .collect(Collectors.toList()); + + List existingGermplasms; + try { + existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGID, program.getId()); + existingGermplasms.forEach(existingGermplasm -> { + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); + }); + return existingGermplasmByGID; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + private Map> initialize_observationUnitByNameNoScope(Program program, List experimentImportRows) { + + Map> observationUnitByNameNoScope = new HashMap<>(); + List uniqueObservationUnitNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpUnitId()) + .distinct() + .collect(Collectors.toList()); + + // check for existing observation units. Don't want to update existing, just create new. + // TODO: do we allow adding observations to existing studies? yes, but not updating + // ignore all data for observation units existing in system + + List existingObservationUnits; + + try { + existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); + existingObservationUnits.forEach(existingObservationUnit -> { + // update mapped brapi import, does in process + observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); + }); + return observationUnitByNameNoScope; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { + Map> studyByNameNoScope = new HashMap<>(); + List uniqueStudyNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnv()) + .distinct() + .collect(Collectors.toList()); + + List existingStudies; + try { + existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies.forEach(existingStudy -> { + String studySequence = null; + if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { + studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); + } + studyByNameNoScope.put( + Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), + new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); + }); + return studyByNameNoScope; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private Map> initialize_uniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); + List uniqueLocationNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getEnvLocation()) + .distinct() + .collect(Collectors.toList()); + + List existingLocations; + try { + existingLocations = brAPILocationDAO.getLocationsByName(uniqueLocationNames, program.getId()); + existingLocations.forEach(existingLocation -> { + locationByName.put(existingLocation.getLocationName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation)); + }); + return locationByName; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } + + private Map> initialize_trialByNameNoScope(Program program, List experimentImportRows) { + Map> trialByNameNoScope = new HashMap<>(); + List uniqueTrialNames = experimentImportRows.stream() + .map(experimentImport -> experimentImport.getExpTitle()) + .distinct() + .collect(Collectors.toList()); + List existingTrials; + try { + existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); + existingTrials.forEach(existingTrial -> { + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); + }); + return trialByNameNoScope; + } catch (ApiException e) { + // We shouldn't get an error back from our services. If we do, nothing the user can do about it + throw new InternalServerException(e.toString(), e); + } + } From ef1d80db079c3debf79485e5731db8b97441e75e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 29 Apr 2022 16:21:03 -0400 Subject: [PATCH 04/24] [BI-1146] refactor. added code for coditionally required cells --- .../ExperimentObservation.java | 2 +- .../importer/services/FileImportService.java | 2 +- .../importer/services/MappingManager.java | 8 +- .../processors/ExperimentProcessor.java | 137 ++++++++++++++---- 4 files changed, 116 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index d32bc6a18..7a2a27b2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -148,7 +148,7 @@ public BrAPIStudy constructBrAPIStudy(Program program, Supplier next study.setStudyType(getExpType()); study.setLocationName(getEnvLocation()); study.setTrialName(getExpTitle()); - study.setSeasons(List.of(getEnvYear())); + study.setSeasons( List.of( getEnvYear()==null ? "" : getEnvYear() ) ); /* TODO: Not used BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 588fcf6a3..ab206e5e6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -427,7 +427,7 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); } catch (ValidatorException e) { - log.error("Validation errors", e); + log.info("Validation errors", e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); progress.setMessage("Multiple Errors"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index 4dd9982e6..9a825dab3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -19,6 +19,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; @@ -46,6 +47,7 @@ import static org.breedinginsight.brapps.importer.model.config.ImportRelationType.DB_LOOKUP_CONSTANT_VALUE; +@Slf4j @Singleton public class MappingManager { @@ -110,8 +112,6 @@ private List map(ImportMapping importMapping, Table data, Map mappings, Table importFile, Integer rowIndex, Map userInput, Boolean process, ValidationErrors validationErrors) throws UnprocessableEntityException { - - Row focusRow = importFile.row(rowIndex); // Process this field ImportFieldType type = field.getAnnotation(ImportFieldType.class); @@ -125,7 +125,6 @@ private void mapField(Object parent, Field field, List mappings, T metadata = field.getAnnotation(ImportFieldMetadata.class) != null ? field.getAnnotation(ImportFieldMetadata.class) : (ImportFieldMetadata) type.clazz().getAnnotation(ImportFieldMetadata.class); } - ImportMappingRequired required = field.getAnnotation(ImportMappingRequired.class); // Check if it is a user input field @@ -136,7 +135,6 @@ private void mapField(Object parent, Field field, List mappings, T } return; } - List foundMappings = new ArrayList<>(); if (mappings != null) { foundMappings = mappings.stream() @@ -151,7 +149,6 @@ private void mapField(Object parent, Field field, List mappings, T } else if (required == null && foundMappings.size() == 0) { return; } - MappingField matchedMapping = foundMappings.get(0); if (type == null) { throw new InternalServerException("BrAPIObject is missing import type annotation"); @@ -310,6 +307,7 @@ private void mapField(Object parent, Field field, List mappings, T // Check non-null value if (required != null && value.isBlank()) { + //throw new UnprocessableEntityException(String.format(blankRequiredField, metadata.name())); ValidationError ve = getMissingRequiredErr(metadata.name()); validationErrors.addError(getRowNumber(rowIndex), ve); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 73b4edfa0..86a531630 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -141,41 +141,35 @@ public Map process(List importRows // For each import row for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); addIfNotNull(GIDCounter, importRow.getGid()); - //////////////////////////////////////////// - // Create and store PendingImportObject's // - //////////////////////////////////////////// - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow ); - mappedImportRow.setTrial( trialPIO ); - this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); - - PendingImportObject locationPIO = createLocationPIO(importRow); - mappedImportRow.setLocation( locationPIO ); - this.locationByName.put( importRow.getEnvLocation(), locationPIO ); - - PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); - mappedImportRow.setStudy( studyPIO ); - this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - mappedImportRow.setObservationUnit(obsUnitPIO); - this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); - - PendingImportObject germplasmPIO = getGidPio(validationErrors, importRow, mappedImportRow); - mappedImportRow.setGermplasm( germplasmPIO ); - validateGerplam( validationErrors, i, germplasmPIO ); + //////////////////////////////////////////////////// + // Create (or get) and store PendingImportObjects // + // into the PendingImport // + //////////////////////////////////////////////////// + PendingImport mappedImportRow = + getPendingImport( + mappedBrAPIImport, + program, + commit, + studyNextVal, + obsUnitNextVal, + i, + importRow); + + validateGerplam( validationErrors, i, mappedImportRow.getGermplasm() ); // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } // End-of-loop + validationErrors = validateConditionallyRequiredFields(importRows, validationErrors); + if (validationErrors.hasErrors() ){ throw new ValidatorException(validationErrors); } @@ -184,6 +178,92 @@ public Map process(List importRows return getStatisticsMap(environmentNameCounter, obsUnitsIDCounter, GIDCounter); } + private ValidationErrors validateConditionallyRequiredFields(List importRows, ValidationErrors validationErrors) { + for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + String experimentTitle = importRow.getExpTitle(); + ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); + boolean isExistingNew = (expState == ImportObjectState.NEW); + + if (isExistingNew) { + String errorMessage = "Field is blank when creating a new experiment"; + validateNotNullOrBlank( importRow.getGid(), + "GID", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpUnit(), + "Exp Unit", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpType(), + "Exp Type", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getEnv(), + "Env", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getEnvLocation(), + "Env Location", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getEnvYear(), + "Env Year", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpUnitId(), + "Exp Unit ID", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpReplicateNo(), + "Exp Replicate #", + errorMessage, validationErrors, i); + validateNotNullOrBlank( importRow.getExpBlockNo(), + "Exp Block #", + errorMessage, validationErrors, i); + } + } + return validationErrors; + } + + private void validateNotNullOrBlank(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { + if ( value==null || value.isBlank()) { + addRowError( + columHeader, + errorMessage, + validationErrors, i + ) ; + } + } + + private void addRowError(String field, String errorMessage, ValidationErrors validationErrors, int i) { + ValidationError ve = new ValidationError(field, errorMessage, HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + } + + private PendingImport getPendingImport( + Map mappedBrAPIImport, + Program program, + boolean commit, + Supplier studyNextVal, + Supplier obsUnitNextVal, + int i, + ExperimentObservation importRow) { + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); + mappedImportRow.setTrial( trialPIO ); + this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); + + PendingImportObject locationPIO = createLocationPIO(importRow); + mappedImportRow.setLocation( locationPIO ); + this.locationByName.put( importRow.getEnvLocation(), locationPIO ); + + PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); + mappedImportRow.setStudy( studyPIO ); + this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); + + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + mappedImportRow.setObservationUnit(obsUnitPIO); + this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + + PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); + mappedImportRow.setGermplasm( germplasmPIO ); + return mappedImportRow; + } + private void addIfNotNull(HashSet set, String setValue) { if( setValue!=null) { set.add(setValue); } } @@ -208,12 +288,12 @@ private Map getStatisticsMap(HashSet en private void validateGerplam(ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { if( germplasmPIO == null ) { - ValidationError ve = new ValidationError("GID", "Not an existing GID", HttpStatus.UNPROCESSABLE_ENTITY); + ValidationError ve = new ValidationError("GID", "A non-existing GID", HttpStatus.UNPROCESSABLE_ENTITY); validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. } } - private PendingImportObject getGidPio(ValidationErrors validationErrors, ExperimentObservation importRow, PendingImport mappedImportRow) { + private PendingImportObject getGidPio(ExperimentObservation importRow, PendingImport mappedImportRow) { if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ return existingGermplasmByGID.get(importRow.getGid()); } @@ -332,6 +412,7 @@ private Map> initialize_observ List uniqueObservationUnitNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getExpUnitId()) .distinct() + .filter(name -> null !=name) .collect(Collectors.toList()); // check for existing observation units. Don't want to update existing, just create new. @@ -358,6 +439,7 @@ private Map> initialize_studyByNameNoSco List uniqueStudyNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getEnv()) .distinct() + .filter(name -> null !=name) .collect(Collectors.toList()); List existingStudies; @@ -384,6 +466,7 @@ private Map> initialize_uniqueLocatio List uniqueLocationNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getEnvLocation()) .distinct() + .filter(name -> null !=name) .collect(Collectors.toList()); List existingLocations; @@ -403,9 +486,11 @@ private Map> initialize_trialByNameNoSco Map> trialByNameNoScope = new HashMap<>(); List uniqueTrialNames = experimentImportRows.stream() .map(experimentImport -> experimentImport.getExpTitle()) - .distinct() + .distinct(). + filter(name -> null !=name) .collect(Collectors.toList()); List existingTrials; +log.info(uniqueTrialNames.toString()); try { existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); From 4b9e37347500b916bb0fe4f029405d1497f1f1ff Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 2 May 2022 10:44:53 -0400 Subject: [PATCH 05/24] [BI-1146] refactor. added getNewBrapiData() --- .../processors/ExperimentProcessor.java | 120 ++++++++---------- 1 file changed, 56 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 86a531630..e243d9762 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -73,12 +73,12 @@ public class ExperimentProcessor implements Processor { private final BrAPIGermplasmDAO brAPIGermplasmDAO; //These are initially populated by the getExistingBrapiData() method, - // then updated by the process() method. + // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; private Map> observationUnitByNameNoScope = null; - // existingGermplasmByGID is initially populated by getExistingBrapiData(), but not updated by the process() method + // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @Inject @@ -119,47 +119,35 @@ public void getExistingBrapiData(List importRows, Program program) * @param mappedBrAPIImport - passed in by reference and modified withing this program (this will latter be passed to the front end for the preview) * @param program * @param user - * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) + * false when used for preview only * @return Map - used to display the summary statistics. * @throws ValidatorException */ @Override - public Map process(List importRows, - Map mappedBrAPIImport, Program program, User user, boolean commit) throws ValidatorException { - //TODO retrieve sequanceName from the program table. - String studySequenceName = ""; - Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); - String opsUnitSequenceName = ""; - Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + public Map process( + List importRows, + Map mappedBrAPIImport, + Program program, + User user, + boolean commit) throws ValidatorException { + ValidationErrors validationErrors = new ValidationErrors(); - // Data for stats. - HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names - HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet GIDCounter = new HashSet<>(); // set of unique GID's + // add "New" pending data to the BrapiData objects + getNewBrapiData(importRows, program, commit); // For each import row for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - // Collect date for stats. - addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); - addIfNotNull(GIDCounter, importRow.getGid()); - - //////////////////////////////////////////////////// - // Create (or get) and store PendingImportObjects // - // into the PendingImport // - //////////////////////////////////////////////////// - PendingImport mappedImportRow = - getPendingImport( - mappedBrAPIImport, - program, - commit, - studyNextVal, - obsUnitNextVal, - i, - importRow); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); + mappedImportRow.setTrial( this.trialByNameNoScope.get( importRow.getExpTitle() ) ); + mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); + mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); + mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( importRow.getExpUnitId() ) ); + PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); + mappedImportRow.setGermplasm( germplasmPIO ); validateGerplam( validationErrors, i, mappedImportRow.getGermplasm() ); @@ -175,7 +163,31 @@ public Map process(List importRows } // Construct our response object - return getStatisticsMap(environmentNameCounter, obsUnitsIDCounter, GIDCounter); + return getStatisticsMap(importRows); + } + + private void getNewBrapiData(List importRows, Program program, boolean commit) { + + //TODO retrieve sequanceName from the program table. + String studySequenceName = ""; + Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); + String opsUnitSequenceName = ""; + Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); + this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); + + PendingImportObject locationPIO = createLocationPIO(importRow); + this.locationByName.put( importRow.getEnvLocation(), locationPIO ); + + PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); + this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); + + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); + this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + } } private ValidationErrors validateConditionallyRequiredFields(List importRows, ValidationErrors validationErrors) { @@ -234,41 +246,21 @@ private void addRowError(String field, String errorMessage, ValidationErrors val validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. } - private PendingImport getPendingImport( - Map mappedBrAPIImport, - Program program, - boolean commit, - Supplier studyNextVal, - Supplier obsUnitNextVal, - int i, - ExperimentObservation importRow) { - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); - mappedImportRow.setTrial( trialPIO ); - this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); - - PendingImportObject locationPIO = createLocationPIO(importRow); - mappedImportRow.setLocation( locationPIO ); - this.locationByName.put( importRow.getEnvLocation(), locationPIO ); - - PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); - mappedImportRow.setStudy( studyPIO ); - this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - mappedImportRow.setObservationUnit(obsUnitPIO); - this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); - - PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); - mappedImportRow.setGermplasm( germplasmPIO ); - return mappedImportRow; - } - private void addIfNotNull(HashSet set, String setValue) { if( setValue!=null) { set.add(setValue); } } - private Map getStatisticsMap(HashSet environmentNameSet, HashSet obsUnitsIDSet, HashSet GIDSet) { + private Map getStatisticsMap(List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet GIDCounter = new HashSet<>(); // set of unique GID's + for (int i = 0; i < importRows.size(); i++) { + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); + addIfNotNull(GIDCounter, importRow.getGid()); + } ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameSet.size()) .build(); From 565be9f07bf6e75ae4f5e73a6e659c43c9a69365 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 3 May 2022 08:40:05 -0400 Subject: [PATCH 06/24] [BI-1146] remove unneeded Classes, clean up code --- .../EnvironmentData.java | 33 ----- .../experimentObservation/ExperimentData.java | 45 ------- .../experimentObservation/FileData.java | 49 ------- .../ObservationUnitData.java | 26 ---- .../processors/ExperimentProcessor.java | 126 ++++++++++-------- 5 files changed, 72 insertions(+), 207 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java deleted file mode 100644 index d67dd74e6..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/EnvironmentData.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.Getter; -import lombok.Setter; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - - -@Getter -@Setter -public class EnvironmentData { - private String env; - private String location; - private String year; - // TODO we don't want a getObservationUnitList() method - private List observationUnitDataList = new ArrayList<>(); - - public EnvironmentData(String env, String location, String year) { - this.env = env; - this.location = location; - this.year = year; - } - - public void addObservationUnitData(ObservationUnitData ou){ - this.observationUnitDataList.add(ou); - } - - public Collection observationUnitValues(){ - return this.observationUnitDataList; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java deleted file mode 100644 index 2be50f798..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentData.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.Getter; -import lombok.Setter; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - - -public class ExperimentData { - @Getter - @Setter - private String title; - @Getter - @Setter - private String description; - - - private Map environments = new HashMap<>(); - - public ExperimentData(String title, String description) { - this.title = title; - this.description = description; - } - - public EnvironmentData retrieve_or_add_environmentData(String envName, String envLoc, String envYear){ - String key = envLoc + envYear; - EnvironmentData environmentData; - if( ! environments.containsKey(key) ){ - environmentData = new EnvironmentData(envName, envLoc, envYear); - environments.put(key, environmentData); - } - else{ - environmentData = this.environments.get(key); - } - return environmentData; - } - - public Collection environmentData_values(){ - return environments.values(); - } -} - - diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java deleted file mode 100644 index 92ea67e4a..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/FileData.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.NoArgsConstructor; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -@NoArgsConstructor -public class FileData { - private Map expMap = new HashMap<>(); - private Map gids = new HashMap<>(); - - public ExperimentData retrieve_or_add_ExperimentData(String title, String description){ - String key = title + description; - ExperimentData experimentData = null; - if( ! expMap.containsKey(key) ){ - experimentData = new ExperimentData(title, description); - expMap.put(key,experimentData); - } - else { - experimentData = expMap.get(key); - } - return experimentData; - } - - public int experiments_size(){ - return expMap.size(); - } - - public ExperimentData get_ExperimentData(String key){ - return expMap.get(key); - } - - public Collection experimentData(){ - return expMap.values(); - } - - public void add_gid(String gid){ - if(!gids.containsKey(gid)){ - gids.put(gid,gid); - } - } - - public int gids_size(){ return gids.size(); } - - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java deleted file mode 100644 index d760c2fee..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ObservationUnitData.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.breedinginsight.brapps.importer.model.imports.experimentObservation; - -import lombok.AccessLevel; -import lombok.Builder; -import lombok.Getter; -import lombok.Setter; -import org.brapi.v2.model.pheno.BrAPIEntryTypeEnum; - -@Builder -@Getter -@Setter -public class ObservationUnitData { - private BrAPIEntryTypeEnum test_or_check = BrAPIEntryTypeEnum.TEST; - - private String germplasm_name = null; - private String gid = null; - private String exp_unit = null; - private String exp_unit_id = null; - private String exp_type = null; - private String exp_replicate_no = null; - private String exp_block_no = null; - private String row = null; - private String column = null; - private String treatment_factors = null; - private String id = null; -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e243d9762..16e7891c3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -21,6 +21,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; @@ -36,7 +37,6 @@ import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -60,8 +60,6 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private FileData fileData = new FileData(); - @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -72,7 +70,7 @@ public class ExperimentProcessor implements Processor { private BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final BrAPIGermplasmDAO brAPIGermplasmDAO; - //These are initially populated by the getExistingBrapiData() method, + //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; private Map> locationByName = null; @@ -116,7 +114,7 @@ public void getExistingBrapiData(List importRows, Program program) /** * @param importRows - one element of the list for every row of the import file. - * @param mappedBrAPIImport - passed in by reference and modified withing this program (this will latter be passed to the front end for the preview) + * @param mappedBrAPIImport - passed in by reference and modified within this program (this will latter be passed to the front end for the preview) * @param program * @param user * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) @@ -149,14 +147,15 @@ public Map process( PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); mappedImportRow.setGermplasm( germplasmPIO ); - validateGerplam( validationErrors, i, mappedImportRow.getGermplasm() ); - + if (! StringUtils.isBlank( importRow.getGid() )) { // if GID is blank, don't bother to check if it is valid. + validateGermplam(importRow,validationErrors, i, germplasmPIO); + } // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); } // End-of-loop - validationErrors = validateConditionallyRequiredFields(importRows, validationErrors); + validationErrors = validateFields(importRows, validationErrors); if (validationErrors.hasErrors() ){ throw new ValidatorException(validationErrors); @@ -190,49 +189,63 @@ private void getNewBrapiData(List importRows, Program program, bool } } - private ValidationErrors validateConditionallyRequiredFields(List importRows, ValidationErrors validationErrors) { + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - String experimentTitle = importRow.getExpTitle(); - ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); - boolean isExistingNew = (expState == ImportObjectState.NEW); - - if (isExistingNew) { - String errorMessage = "Field is blank when creating a new experiment"; - validateNotNullOrBlank( importRow.getGid(), - "GID", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpUnit(), - "Exp Unit", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpType(), - "Exp Type", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getEnv(), - "Env", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getEnvLocation(), - "Env Location", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getEnvYear(), - "Env Year", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpUnitId(), - "Exp Unit ID", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpReplicateNo(), - "Exp Replicate #", - errorMessage, validationErrors, i); - validateNotNullOrBlank( importRow.getExpBlockNo(), - "Exp Block #", - errorMessage, validationErrors, i); - } + + validateConditionallyRequired(validationErrors, i, importRow); } return validationErrors; } - private void validateNotNullOrBlank(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { - if ( value==null || value.isBlank()) { + private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { + String experimentTitle = importRow.getExpTitle(); + String obsUnitID = importRow.getObsUnitID(); + if( StringUtils.isBlank( obsUnitID )){ + validateRequiredCell( + experimentTitle, + "Exp Title", + "Field is blank", validationErrors, i + ); + } + + ImportObjectState expState = this.trialByNameNoScope.get(experimentTitle).getState(); + boolean isExperimentNew = (expState == ImportObjectState.NEW); + + if (isExperimentNew) { + String errorMessage = "Field is blank when creating a new experiment"; + validateRequiredCell( importRow.getGid(), + "GID", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpUnit(), + "Exp Unit", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpType(), + "Exp Type", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getEnv(), + "Env", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getEnvLocation(), + "Env Location", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getEnvYear(), + "Env Year", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpUnitId(), + "Exp Unit ID", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpReplicateNo(), + "Exp Replicate #", + errorMessage, validationErrors, i); + validateRequiredCell( importRow.getExpBlockNo(), + "Exp Block #", + errorMessage, validationErrors, i); + } + } + + private void validateRequiredCell(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { + if ( StringUtils.isBlank( value )) { addRowError( columHeader, errorMessage, @@ -254,21 +267,22 @@ private Map getStatisticsMap(List // Data for stats. HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet GIDCounter = new HashSet<>(); // set of unique GID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's for (int i = 0; i < importRows.size(); i++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); - addIfNotNull(GIDCounter, importRow.getGid()); + addIfNotNull(gidCounter, importRow.getGid()); } ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameSet.size()) + .newObjectCount(environmentNameCounter.size()) .build(); ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDSet.size()) + .newObjectCount(obsUnitsIDCounter.size()) .build(); ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(GIDSet.size()) + .newObjectCount(gidCounter.size()) .build(); return Map.of( @@ -278,10 +292,14 @@ private Map getStatisticsMap(List ); } - private void validateGerplam(ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { - if( germplasmPIO == null ) { - ValidationError ve = new ValidationError("GID", "A non-existing GID", HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(i + 2, ve); // +2 instead of +1 to account for the column header row. + private void validateGermplam(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { + // error if GID is not blank but GID dose not already exist + if( !StringUtils.isBlank( importRow.getGid()) && germplasmPIO == null ) { + addRowError( + "GID", + "A non-existing GID", + validationErrors, i + ); } } From 15c37bef0ed57f33506e00818f8c902a2dcdf16f Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 12:31:38 -0400 Subject: [PATCH 07/24] [BI-1146] added validation for unique observation Unit Id --- .../services/processors/ExperimentProcessor.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 16e7891c3..541160028 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -75,6 +75,8 @@ public class ExperimentProcessor implements Processor { private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; + // It is assumed that there are no pre-existing observation units for the given environment (so this will not be + // intitalized by getExistingBrapiData private Map> observationUnitByNameNoScope = null; // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -190,14 +192,26 @@ private void getNewBrapiData(List importRows, Program program, bool } private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { + HashSet uniqueStudyAndObsUnit = new HashSet<>(); for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); - validateConditionallyRequired(validationErrors, i, importRow); + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, i, importRow); } return validationErrors; } + private void validateUniqueObsUnits(ValidationErrors validationErrors, HashSet uniqueStudyAndObsUnit, int i, ExperimentObservation importRow) { + String envIdPlusStudyId = importRow.getEnv() + importRow.getExpUnitId(); + if( uniqueStudyAndObsUnit.contains( envIdPlusStudyId )){ + String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); + this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); + } + else{ + uniqueStudyAndObsUnit.add( envIdPlusStudyId ); + } + } + private void validateConditionallyRequired(ValidationErrors validationErrors, int i, ExperimentObservation importRow) { String experimentTitle = importRow.getExpTitle(); String obsUnitID = importRow.getObsUnitID(); From 3facc3df83377c550df4f6e4756470aabdce6133 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 16:45:36 -0400 Subject: [PATCH 08/24] [BI-1146] Fixed bug. If spreadsheet cell is 'T', tableSaw assumes that it is a BOOLEAN --- .../brapps/importer/services/MappingManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index 9a825dab3..cecb9688f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -281,6 +281,9 @@ private void mapField(Object parent, Field field, List mappings, T } else if (columnType == ColumnType.INTEGER) { fileValue = String.valueOf(focusRow.getInt(matchedMapping.getValue().getFileFieldName())); } + else if (columnType == ColumnType.BOOLEAN) { + fileValue = String.valueOf(focusRow.getBoolean(matchedMapping.getValue().getFileFieldName())); + } else { fileValue = focusRow.getString(matchedMapping.getValue().getFileFieldName()); } From 0f2171962f3529a957dc3036e93eaa5982ad6c78 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 10 May 2022 09:43:37 -0400 Subject: [PATCH 09/24] [BI-1189] WIP --- .../processors/ExperimentProcessor.java | 69 ++++++++++++++++++- .../services/processors/StudyProcessor.java | 2 +- .../services/ProgramService.java | 6 ++ .../V0.5.36__create_study_sequence.sql | 34 +++++++++ 4 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 src/main/resources/db/migration/V0.5.36__create_study_sequence.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 541160028..4b85f16d4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -19,6 +19,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -170,7 +171,11 @@ public Map process( private void getNewBrapiData(List importRows, Program program, boolean commit) { //TODO retrieve sequanceName from the program table. - String studySequenceName = ""; + String studySequenceName = program.getStudySequence(); + if (studySequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the study sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for study import"); + } Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); String opsUnitSequenceName = ""; Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); @@ -382,10 +387,70 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { + List trials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List locations = ProcessorData.getNewObjects(this.locationByName); + List studies = ProcessorData.getNewObjects(this.studyByNameNoScope); + + List createdTrials = new ArrayList<>(); + List createdLocations = new ArrayList<>(); + List createdStudies = new ArrayList<>(); + try { + createdTrials.addAll(brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload)); + createdLocations.addAll(brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload)); + + updateStudyDependencyValues(mappedBrAPIImport); + createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload)); + } catch (ApiException e) { + throw new InternalServerException(e.toString(), e); + } + + // Update our records + createdTrials.forEach(trial -> { + PendingImportObject preview = this.trialByNameNoScope.get(trial.getTrialName()); + preview.setBrAPIObject(trial); + }); + createdLocations.forEach(location -> { + PendingImportObject preview = locationByName.get(location.getLocationName()); + preview.setBrAPIObject(location); + }); + createdStudies.forEach(study -> { + PendingImportObject preview = this.studyByNameNoScope.get(study.getStudyName()); + preview.setBrAPIObject(study); + }); } private void updateDependencyValues(Map mappedBrAPIImport) { - // TODO + updateStudyDependencyValues(mappedBrAPIImport); + } + + private void updateStudyDependencyValues(Map mappedBrAPIImport) { + // update location DbIds in studies for all distinct locations + mappedBrAPIImport.values().stream() + .map(PendingImport::getLocation) + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateStudyLocationDbId); + + // update trial DbIds in studies for all distinct trials + mappedBrAPIImport.values().stream() + .map(PendingImport::getTrial) + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateTrialDbId); + } + + private void updateStudyLocationDbId(BrAPILocation location) { + this.studyByNameNoScope.values().stream() + .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getLocationName())) + .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); + } + + private void updateTrialDbId(BrAPITrial trial) { + this.studyByNameNoScope.values().stream() + .filter(study -> study.getBrAPIObject().getTrialName().equals(trial.getTrialName())) + .forEach(study -> study.getBrAPIObject().setTrialDbId(trial.getTrialDbId())); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index ff9b1d6ea..39cca38a8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -157,7 +157,7 @@ private void updateDependencyValues(Map mappedBrAPIImpor } private void updateLocationDbId(BrAPILocation location) { - studyByName.values().stream() + this.studyByName.values().stream() .filter(study -> study.getBrAPIObject().getLocationName().equals(location.getLocationName())) .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); } diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index e482fdf79..5eb1ac2fa 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -59,6 +59,7 @@ public class ProgramService { private static final String PROGRAM_NAME_IN_USE = "PROGRAM_NAME_IN_USE"; private static final String PROGRAM_KEY_IN_USE = "PROGRAM_KEY_IN_USE"; private static final String GERMPLASM_SEQUENCE_TEMPLATE = "%s_germplasm_sequence"; + private static final String STUDY_SEQUENCE_TEMPLATE = "%s_study_sequence"; @Inject public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, ProgramObservationLevelDAO programObservationLevelDAO, @@ -161,6 +162,10 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse String germplasm_sequence_name = String.format(GERMPLASM_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(germplasm_sequence_name).execute(); + // Create study sequence + String study_sequence_name = String.format(STUDY_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); + dsl.createSequence(study_sequence_name).execute(); + // Parse and create the program object ProgramEntity programEntity = ProgramEntity.builder() .name(programRequest.getName()) @@ -171,6 +176,7 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .brapiUrl(brapiUrl) .key(programRequest.getKey()) .germplasmSequence(germplasm_sequence_name) + .studySequence(study_sequence_name) .createdBy(actingUser.getId()) .updatedBy(actingUser.getId()) .build(); diff --git a/src/main/resources/db/migration/V0.5.36__create_study_sequence.sql b/src/main/resources/db/migration/V0.5.36__create_study_sequence.sql new file mode 100644 index 000000000..04c2f11b4 --- /dev/null +++ b/src/main/resources/db/migration/V0.5.36__create_study_sequence.sql @@ -0,0 +1,34 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table program add column study_sequence text; + +do +$$ +declare +f record; +begin +for f in select * from program +loop + if f.key is NULL then + RAISE EXCEPTION 'Programs must have a key associated with them'; + end if; + execute format('create sequence %s_study_sequence',f.key); + update program set study_sequence = format('%s_study_sequence', f.key) where id = f.id; +end loop; +end; +$$ \ No newline at end of file From 2520ecccf57678366ec1d1468bf5a318af11b889 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 12 May 2022 08:28:46 -0400 Subject: [PATCH 10/24] [BI-1189] added studySequence to Program model object --- src/main/java/org/breedinginsight/model/Program.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 35cebadb5..63ab63d65 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -86,6 +86,7 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .updatedBy(record.getValue(programTable.UPDATED_BY)) .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) + .studySequence(record.getValue(programTable.STUDY_SEQUENCE)) .build(); return program; From d2193e86ab5ad32e1bcd1dbf5780a8b21b02d273 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 17 May 2022 13:40:56 -0400 Subject: [PATCH 11/24] BI-1189 WIP --- .../ExperimentObservation.java | 6 +- .../processors/ExperimentProcessor.java | 115 ++++++++++++++---- .../org/breedinginsight/model/Program.java | 1 + .../services/ProgramService.java | 5 + .../breedinginsight/utilities/Utilities.java | 6 + .../V0.5.37__create_obs_unit_sequence.sql | 34 ++++++ 6 files changed, 139 insertions(+), 28 deletions(-) create mode 100644 src/main/resources/db/migration/V0.5.37__create_obs_unit_sequence.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 7a2a27b2b..61ddec164 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -74,7 +74,8 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id="envLocation", name="Environment Location", description = "Location of the environment") private String envLocation; - @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) + //TODO make this ImportFieldTypeEnum.INTEGER + @ImportFieldType(type= ImportFieldTypeEnum.TEXT) @ImportFieldMetadata(id="envYear", name="Environment Year", description = "Year corresponding to the environment") private String envYear; @@ -149,6 +150,7 @@ public BrAPIStudy constructBrAPIStudy(Program program, Supplier next study.setLocationName(getEnvLocation()); study.setTrialName(getExpTitle()); study.setSeasons( List.of( getEnvYear()==null ? "" : getEnvYear() ) ); + /* TODO: Not used BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); @@ -171,7 +173,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(Program program, Suppl observationUnit.setStudyName(getEnv()); // TODO: Set the germplasm - //observationUnit.setGermplasmName(getGermplasm().getReferenceValue()); + observationUnit.setGermplasmName(getGermplasmName()); BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 4b85f16d4..e8d1c6981 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -177,8 +177,14 @@ private void getNewBrapiData(List importRows, Program program, bool throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for study import"); } Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); - String opsUnitSequenceName = ""; + + String opsUnitSequenceName = program.getObsUnitSequence(); + if (opsUnitSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the obsUnit sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); + } Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + for (int i = 0; i < importRows.size(); i++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); @@ -190,7 +196,6 @@ private void getNewBrapiData(List importRows, Program program, bool PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); - PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); } @@ -333,7 +338,7 @@ private PendingImportObject getGidPio(ExperimentObservation impo private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { PendingImportObject pio = null; - if( observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { + if( this.observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { pio = observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ; } else{ @@ -390,40 +395,97 @@ public void postBrapiData(Map mappedBrAPIImport, Program List trials = ProcessorData.getNewObjects(this.trialByNameNoScope); List locations = ProcessorData.getNewObjects(this.locationByName); List studies = ProcessorData.getNewObjects(this.studyByNameNoScope); + List observationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); List createdTrials = new ArrayList<>(); List createdLocations = new ArrayList<>(); List createdStudies = new ArrayList<>(); + List createdObservationUnits = new ArrayList<>(); try { createdTrials.addAll(brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload)); + for ( BrAPITrial createdTrial: createdTrials ) { + String createdTrialName = createdTrial.getTrialName(); + String createdTrialName_no_key = Utilities.removeProgramKey( createdTrialName, program.getKey() ); + PendingImportObject pi = this.trialByNameNoScope.get(createdTrialName_no_key); + BrAPITrial listedTrial = pi.getBrAPIObject(); + String dbid = createdTrial.getTrialDbId(); + listedTrial.setTrialDbId( dbid ); + } createdLocations.addAll(brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload)); - updateStudyDependencyValues(mappedBrAPIImport); + updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload)); + for( BrAPIStudy createdStudy : createdStudies){ + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData( createdStudy.getStudyName(), program.getKey() ); + PendingImportObject pi = this.studyByNameNoScope.get( createdStudy_name_no_key ); + BrAPIStudy brAPIStudy = pi.getBrAPIObject(); + brAPIStudy.setStudyDbId( createdStudy.getStudyDbId() ); + } + + updateObsUnitDependencyValues(mappedBrAPIImport,program.getKey()); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } - // Update our records - createdTrials.forEach(trial -> { - PendingImportObject preview = this.trialByNameNoScope.get(trial.getTrialName()); - preview.setBrAPIObject(trial); - }); - createdLocations.forEach(location -> { - PendingImportObject preview = locationByName.get(location.getLocationName()); - preview.setBrAPIObject(location); - }); - createdStudies.forEach(study -> { - PendingImportObject preview = this.studyByNameNoScope.get(study.getStudyName()); - preview.setBrAPIObject(study); - }); +// // Update our records +// createdTrials.forEach(trial -> { +// PendingImportObject preview = this.trialByNameNoScope.get( Utilities.removeProgramKey( trial.getTrialName(),program.getKey() ) ); +// preview.setBrAPIObject(trial); +// }); +// createdLocations.forEach(location -> { +// PendingImportObject preview = locationByName.get(location.getLocationName()); +// preview.setBrAPIObject(location); +// }); +// createdStudies.forEach(study -> { +// String studyName = Utilities.removeProgramKey(study.getStudyName(),program.getKey() ); +// PendingImportObject preview = this.studyByNameNoScope.get( studyName ) ; +// preview.setBrAPIObject(study); +// }); } - private void updateDependencyValues(Map mappedBrAPIImport) { - updateStudyDependencyValues(mappedBrAPIImport); +// private void updateDependencyValues(Map mappedBrAPIImport) { +// updateStudyDependencyValues(mappedBrAPIImport); +// } + + private void updateObsUnitDependencyValues(Map mappedBrAPIImport, String programKey) { + + // update study DbIds + this.studyByNameNoScope.values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); + + // update germplasm DbIds + this.existingGermplasmByGID.values().stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); + } + + private void updateStudyDbId(BrAPIStudy study, String programKey) { + this.observationUnitByNameNoScope.values().stream() + .filter(obsUnit -> obsUnit.getBrAPIObject().getStudyName().equals( Utilities.removeProgramKeyAndUnknownAdditionalData( study.getStudyName(), programKey ) )) + .forEach(obsUnit -> obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId())); } - private void updateStudyDependencyValues(Map mappedBrAPIImport) { +// private void updateGermplasmDbId(BrAPIGermplasm germplasm) { +// this.observationUnitByNameNoScope.values().stream() +// .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && +// obsUnit.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) +// .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); +// } + + private void updateGermplasmDbId(BrAPIGermplasm germplasm) { + this.observationUnitByNameNoScope.values().stream() + .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && + obsUnit.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) + .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); + } + + + private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations mappedBrAPIImport.values().stream() .map(PendingImport::getLocation) @@ -433,12 +495,11 @@ private void updateStudyDependencyValues(Map mappedBrAPI .forEach(this::updateStudyLocationDbId); // update trial DbIds in studies for all distinct trials - mappedBrAPIImport.values().stream() - .map(PendingImport::getTrial) + this.trialByNameNoScope.values().stream() .filter(Objects::nonNull) .distinct() .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateTrialDbId); + .forEach(trait -> this.updateTrialDbId(trait, programKey)); } private void updateStudyLocationDbId(BrAPILocation location) { @@ -447,9 +508,9 @@ private void updateStudyLocationDbId(BrAPILocation location) { .forEach(study -> study.getBrAPIObject().setLocationDbId(location.getLocationDbId())); } - private void updateTrialDbId(BrAPITrial trial) { + private void updateTrialDbId(BrAPITrial trial, String programKey) { this.studyByNameNoScope.values().stream() - .filter(study -> study.getBrAPIObject().getTrialName().equals(trial.getTrialName())) + .filter(study -> study.getBrAPIObject().getTrialName().equals(Utilities.removeProgramKey(trial.getTrialName(), programKey ) ) ) .forEach(study -> study.getBrAPIObject().setTrialDbId(trial.getTrialDbId())); } @@ -594,7 +655,9 @@ private Map> initialize_trialByNameNoSco throw new InternalServerException(e.toString(), e); } } - + private String simpleStudyName(String scopedName){ + return scopedName.replaceFirst(" \\[.*\\]", ""); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 63ab63d65..3d84ca31c 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -87,6 +87,7 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) .studySequence(record.getValue(programTable.STUDY_SEQUENCE)) + .obsUnitSequence(record.getValue(programTable.OBS_UNIT_SEQUENCE)) .build(); return program; diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 5eb1ac2fa..bbedaa8af 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -60,6 +60,7 @@ public class ProgramService { private static final String PROGRAM_KEY_IN_USE = "PROGRAM_KEY_IN_USE"; private static final String GERMPLASM_SEQUENCE_TEMPLATE = "%s_germplasm_sequence"; private static final String STUDY_SEQUENCE_TEMPLATE = "%s_study_sequence"; + private static final String OBS_UNIT_SEQUENCE_TEMPLATE = "%s_obs_unit_sequence"; @Inject public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, ProgramObservationLevelDAO programObservationLevelDAO, @@ -166,6 +167,10 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse String study_sequence_name = String.format(STUDY_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(study_sequence_name).execute(); + // Create obs unit sequence + String obs_unit_sequence_name = String.format(OBS_UNIT_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); + dsl.createSequence(obs_unit_sequence_name).execute(); + // Parse and create the program object ProgramEntity programEntity = ProgramEntity.builder() .name(programRequest.getName()) diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index ee992e764..343e0beab 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -91,4 +91,10 @@ public static String removeProgramKey(String original, String programKey, String public static String removeProgramKey(String original, String programKey) { return removeProgramKey(original, programKey, null); } + + public static String removeProgramKeyAndUnknownAdditionalData(String original, String programKey) { + String keyValueRegEx = String.format(" \\[%s\\-.*\\]", programKey); + String stripped = original.replaceAll(keyValueRegEx, ""); + return stripped; + } } diff --git a/src/main/resources/db/migration/V0.5.37__create_obs_unit_sequence.sql b/src/main/resources/db/migration/V0.5.37__create_obs_unit_sequence.sql new file mode 100644 index 000000000..f0bed3308 --- /dev/null +++ b/src/main/resources/db/migration/V0.5.37__create_obs_unit_sequence.sql @@ -0,0 +1,34 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table program add column obs_unit_sequence text; + +do +$$ +declare +f record; +begin +for f in select * from program +loop + if f.key is NULL then + RAISE EXCEPTION 'Programs must have a key associated with them'; + end if; + execute format('create sequence %s_obs_unit_sequence',f.key); + update program set obs_unit_sequence = format('%s_obs_unit_sequence', f.key) where id = f.id; +end loop; +end; +$$ \ No newline at end of file From 84fb5a1b189b9afde336da6f8fa5cb4e92e9bb5e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 23 May 2022 12:37:33 -0400 Subject: [PATCH 12/24] [BI-1189] added external references --- .../ExperimentObservation.java | 75 +++++++++++++++---- .../model/response/PendingImportObject.java | 7 +- .../processors/ExperimentProcessor.java | 45 +++++++---- .../org/breedinginsight/model/Program.java | 2 +- .../services/ProgramService.java | 1 + .../breedinginsight/utilities/Utilities.java | 4 + 6 files changed, 101 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 61ddec164..43690bff9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -17,9 +17,12 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; +import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Property; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapps.importer.model.config.*; @@ -107,11 +110,13 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id="ObsUnitID", name="Observation Unit ID", description = "A database generated unique identifier for experimental observation units") private String ObsUnitID; - public BrAPITrial constructBrAPITrial(Program program, boolean commit) { + public BrAPITrial constructBrAPITrial(Program program, boolean commit, String referenceSource, UUID id) { BrAPIProgram brapiProgram = program.getBrapiProgram(); BrAPITrial trial = new BrAPITrial(); if( commit ){ - trial.setTrialName( getTrialNameWithProgramKey( program )); + trial.setTrialName( Utilities.appendProgramKey(getExpTitle(), program.getKey() )); + // Set external reference + trial.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, id, null, null)); } else{ trial.setTrialName( getExpTitle() ); @@ -124,11 +129,8 @@ public BrAPITrial constructBrAPITrial(Program program, boolean commit) { // TODO: Get to a constant trial.putAdditionalInfoItem("defaultObservationLevel", getExpUnit()); trial.putAdditionalInfoItem("experimentType", getExpType()); - return trial; - } - public String getTrialNameWithProgramKey(Program program){ - return String.format("%s [%s]", getExpTitle(), program.getKey()); + return trial; } public BrAPILocation constructBrAPILocation() { @@ -137,10 +139,18 @@ public BrAPILocation constructBrAPILocation() { return location; } - public BrAPIStudy constructBrAPIStudy(Program program, Supplier nextVal, boolean commit) { + public BrAPIStudy constructBrAPIStudy( + Program program, + Supplier nextVal, + boolean commit, + String referenceSource, + UUID trialId, + UUID id) { BrAPIStudy study = new BrAPIStudy(); if ( commit ){ study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), nextVal.get().toString())); + // Set external reference + study.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialId, id,null)); } else { study.setStudyName(getEnv()); @@ -151,21 +161,32 @@ public BrAPIStudy constructBrAPIStudy(Program program, Supplier next study.setTrialName(getExpTitle()); study.setSeasons( List.of( getEnvYear()==null ? "" : getEnvYear() ) ); - /* - TODO: Not used + String designType = "Analysis"; BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); - design.setPUI(getExperimentalDesignPUI()); + design.setPUI(designType); + design.setDescription(designType); study.setExperimentalDesign(design); - */ return study; } - public BrAPIObservationUnit constructBrAPIObservationUnit(Program program, Supplier nextVal, boolean commit) { + public BrAPIObservationUnit constructBrAPIObservationUnit( + Program program, + Supplier nextVal, + boolean commit, + String germplasmName, + String referenceSource, + UUID trialID, + UUID studyID, + UUID id + ) { BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), id.toString())); + + // Set external reference + observationUnit.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialID, studyID, id)); } else { observationUnit.setObservationUnitName(getExpUnitId()); @@ -173,12 +194,17 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(Program program, Suppl observationUnit.setStudyName(getEnv()); // TODO: Set the germplasm - observationUnit.setGermplasmName(getGermplasmName()); + if(germplasmName==null){ + germplasmName = getGermplasmName(); + } + observationUnit.setGermplasmName(germplasmName); BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName(getExpUnit()); + level.setLevelName("plot"); + level.setLevelCode( getExpUnitId() ); position.setObservationLevel(level); + observationUnit.putAdditionalInfoItem("observationLevel", getExpUnit()); // Exp Unit List levelRelationships = new ArrayList<>(); @@ -226,4 +252,23 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(Program program, Suppl return observationUnit; } + private List getBrAPIExternalReferences( + Program program, String referenceSource, UUID trialId, UUID studyId, UUID obsUnitId) { + List refs = new ArrayList<>(); + + addReference(refs, referenceSource, program.getId(), "programs"); + if( trialId != null ) { addReference(refs, referenceSource, trialId, "trials"); } + if( studyId != null ) { addReference(refs, referenceSource, studyId, "studies"); } + if( obsUnitId != null ) { addReference(refs, referenceSource, obsUnitId, "observationunits"); } + + return refs; + } + + private void addReference(List refs, String referenceSource, UUID uuid, String refSourceName) { + BrAPIExternalReference reference; + reference = new BrAPIExternalReference(); + reference.setReferenceSource( String.format("%s/%s",referenceSource, refSourceName) ); + reference.setReferenceID(uuid.toString()); + refs.add(reference); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java index 9bdad5570..c7619df80 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java @@ -30,9 +30,12 @@ public class PendingImportObject { private T brAPIObject; private UUID id; - public PendingImportObject(ImportObjectState state, T brAPIObject) { + public PendingImportObject(ImportObjectState state, T brAPIObject, UUID id) { this.state = state; this.brAPIObject = brAPIObject; - this.id = UUID.randomUUID(); + this.id = id; + } + public PendingImportObject(ImportObjectState state, T brAPIObject) { + this(state, brAPIObject, UUID.randomUUID()); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e8d1c6981..9a4c226e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -196,6 +196,7 @@ private void getNewBrapiData(List importRows, Program program, bool PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); + PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); } @@ -342,7 +343,16 @@ private PendingImportObject createObsUnitPIO(Program progr pio = observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ; } else{ - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit); + String germplasmName = ""; + if( this.existingGermplasmByGID.get( importRow.getGid() ) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()).getBrAPIObject().getGermplasmName(); + } + PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, obsUnitNextVal, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit); } return pio; @@ -354,8 +364,11 @@ private PendingImportObject createStudyPIO(Program program, boolean pio = studyByNameNoScope.get( importRow.getEnv() ) ; } else{ - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit); - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy); + PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit, BRAPI_REFERENCE_SOURCE, trialID, id); + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); } return pio; } @@ -378,8 +391,9 @@ private PendingImportObject createTrialPIO(Program program, boolean pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; } else { - BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit); - pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial); + UUID id = UUID.randomUUID(); + BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit, BRAPI_REFERENCE_SOURCE, id); + pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); } return pio; } @@ -397,12 +411,9 @@ public void postBrapiData(Map mappedBrAPIImport, Program List studies = ProcessorData.getNewObjects(this.studyByNameNoScope); List observationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); - List createdTrials = new ArrayList<>(); - List createdLocations = new ArrayList<>(); - List createdStudies = new ArrayList<>(); - List createdObservationUnits = new ArrayList<>(); try { - createdTrials.addAll(brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload)); + List createdTrials = new ArrayList<>(); + createdTrials.addAll( brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload) ); for ( BrAPITrial createdTrial: createdTrials ) { String createdTrialName = createdTrial.getTrialName(); String createdTrialName_no_key = Utilities.removeProgramKey( createdTrialName, program.getKey() ); @@ -411,10 +422,11 @@ public void postBrapiData(Map mappedBrAPIImport, Program String dbid = createdTrial.getTrialDbId(); listedTrial.setTrialDbId( dbid ); } - createdLocations.addAll(brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload)); + brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload); updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); - createdStudies.addAll(brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload)); + List createdStudies = new ArrayList<>(); + createdStudies.addAll( brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload) ); for( BrAPIStudy createdStudy : createdStudies){ String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData( createdStudy.getStudyName(), program.getKey() ); PendingImportObject pi = this.studyByNameNoScope.get( createdStudy_name_no_key ); @@ -423,7 +435,9 @@ public void postBrapiData(Map mappedBrAPIImport, Program } updateObsUnitDependencyValues(mappedBrAPIImport,program.getKey()); + brAPIObservationUnitDAO.createBrAPIObservationUnits(observationUnits, program.getId(), upload); } catch (ApiException e) { + log.error(e.getResponseBody().toString()); throw new InternalServerException(e.toString(), e); } @@ -539,14 +553,14 @@ private ArrayList getGermplasmByAccessionNumber( private Map> initialize_existingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); - List uniqueGermplasmGID = experimentImportRows.stream() + List uniqueGermplasmGIDs = experimentImportRows.stream() .map(experimentImport -> experimentImport.getGid()) .distinct() .collect(Collectors.toList()); List existingGermplasms; try { - existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGID, program.getId()); + existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); existingGermplasms.forEach(existingGermplasm -> { existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); }); @@ -634,8 +648,9 @@ private Map> initialize_uniqueLocatio private Map> initialize_trialByNameNoScope(Program program, List experimentImportRows) { Map> trialByNameNoScope = new HashMap<>(); + String programKey = program.getKey(); List uniqueTrialNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getExpTitle()) + .map(experimentImport -> Utilities.appendProgramKey( experimentImport.getExpTitle(), programKey, null) ) .distinct(). filter(name -> null !=name) .collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 3d84ca31c..e69cc6f8f 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -87,7 +87,7 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) .studySequence(record.getValue(programTable.STUDY_SEQUENCE)) - .obsUnitSequence(record.getValue(programTable.OBS_UNIT_SEQUENCE)) + .obsUnitSequence(( record.getValue(programTable.OBS_UNIT_SEQUENCE))) .build(); return program; diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index bbedaa8af..1ac0357cc 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -182,6 +182,7 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .key(programRequest.getKey()) .germplasmSequence(germplasm_sequence_name) .studySequence(study_sequence_name) + .obsUnitSequence( obs_unit_sequence_name ) .createdBy(actingUser.getId()) .updatedBy(actingUser.getId()) .build(); diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 343e0beab..aaa44518a 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -63,6 +63,10 @@ public static String appendProgramKey(String original, String programKey, String } } + public static String appendProgramKey( String original, String programKey ){ + return appendProgramKey( original, programKey, null); + } + /** * Remove program key from a string. Returns a new value instead of altering original string. * From 21c775d14dbe3eb353c26ab594b4b8277d5ae202 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 25 May 2022 15:39:57 -0400 Subject: [PATCH 13/24] [BI-1189] fixed bugs, convert year to season DbId --- .../controllers/UploadController.java | 3 +- .../brapps/importer/daos/BrAPISeasonDAO.java | 73 ++++++++ .../ExperimentObservation.java | 24 +-- .../processors/ExperimentProcessor.java | 170 +++++++++--------- 4 files changed, 177 insertions(+), 93 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index fa18760fb..f1be7e6ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -35,6 +35,7 @@ import org.breedinginsight.brapps.importer.services.FileImportService; import org.breedinginsight.services.exceptions.*; +import javax.annotation.Nullable; import javax.inject.Inject; import java.util.Map; import java.util.UUID; @@ -110,7 +111,7 @@ public HttpResponse> getUploadData(@PathVariable UUID p @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID uploadId, @Body Map userInput) { + @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, userInput, true); diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java new file mode 100644 index 000000000..2685b0245 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java @@ -0,0 +1,73 @@ +package org.breedinginsight.brapps.importer.daos; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.core.SeasonQueryParams; +import org.brapi.client.v2.modules.core.ListsApi; +import org.brapi.client.v2.modules.core.SeasonsApi; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIResponse; +import org.brapi.v2.model.BrAPIResponseResult; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.request.BrAPIListSearchRequest; +import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; +import org.brapi.v2.model.core.response.BrAPISeasonListResponse; +import org.brapi.v2.model.core.response.BrAPISeasonListResponseResult; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.utilities.BrAPIDAOUtil; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +@Slf4j +public class BrAPISeasonDAO { + + private ProgramDAO programDAO; + private ImportDAO importDAO; + + @Inject + public BrAPISeasonDAO(ProgramDAO programDAO, ImportDAO importDAO) { + this.programDAO = programDAO; + this.importDAO = importDAO; + } + + public List getSeasonByYear(String year, UUID programId) throws ApiException { + SeasonsApi api = new SeasonsApi(programDAO.getCoreClient(programId)); + SeasonQueryParams queryParams = + SeasonQueryParams.builder() + .year( year ) + .build(); + + List seasons = new ArrayList<>(); + ApiResponse apiResponse = api.seasonsGet( queryParams ); + BrAPISeasonListResponse seasonListResponse = apiResponse.getBody(); + BrAPISeasonListResponseResult result = seasonListResponse.getResult(); + seasons = result.getData(); + + return seasons; + } + + public BrAPISeason addOneSeason(BrAPISeason season, UUID programId) throws ApiException { + BrAPISeason resultSeason = null; + SeasonsApi api = new SeasonsApi(programDAO.getCoreClient(programId)); + + ApiResponse apiResponse = api.seasonsPost(Arrays.asList(season)); + BrAPISeasonListResponse seasonListResponse = apiResponse.getBody(); + BrAPISeasonListResponseResult result = seasonListResponse.getResult(); + List seasons = result.getData(); + if (seasons.size() > 0) { + resultSeason = seasons.get(0); + } + return resultSeason; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 43690bff9..b9ce5f441 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -17,8 +17,6 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; -import com.google.gson.JsonObject; -import io.micronaut.context.annotation.Property; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -115,6 +113,7 @@ public BrAPITrial constructBrAPITrial(Program program, boolean commit, String re BrAPITrial trial = new BrAPITrial(); if( commit ){ trial.setTrialName( Utilities.appendProgramKey(getExpTitle(), program.getKey() )); + // Set external reference trial.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, id, null, null)); } @@ -141,14 +140,14 @@ public BrAPILocation constructBrAPILocation() { public BrAPIStudy constructBrAPIStudy( Program program, - Supplier nextVal, boolean commit, String referenceSource, UUID trialId, UUID id) { BrAPIStudy study = new BrAPIStudy(); if ( commit ){ - study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), nextVal.get().toString())); + study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), trialId.toString())); + // Set external reference study.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialId, id,null)); } @@ -183,7 +182,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); if( commit){ - observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), id.toString())); + observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); // Set external reference observationUnit.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialID, studyID, id)); @@ -253,22 +252,23 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } private List getBrAPIExternalReferences( - Program program, String referenceSource, UUID trialId, UUID studyId, UUID obsUnitId) { + Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { List refs = new ArrayList<>(); - addReference(refs, referenceSource, program.getId(), "programs"); - if( trialId != null ) { addReference(refs, referenceSource, trialId, "trials"); } - if( studyId != null ) { addReference(refs, referenceSource, studyId, "studies"); } - if( obsUnitId != null ) { addReference(refs, referenceSource, obsUnitId, "observationunits"); } + addReference(refs, program.getId(), referenceSourceBaseName, "programs"); + if( trialId != null ) { addReference(refs, trialId, referenceSourceBaseName, "trials"); } + if( studyId != null ) { addReference(refs, studyId, referenceSourceBaseName, "studies"); } + if( obsUnitId != null ) { addReference(refs, obsUnitId, referenceSourceBaseName, "observationunits"); } return refs; } - private void addReference(List refs, String referenceSource, UUID uuid, String refSourceName) { + private void addReference(List refs, UUID uuid, String referenceBaseNameSource, String refSourceName) { BrAPIExternalReference reference; reference = new BrAPIExternalReference(); - reference.setReferenceSource( String.format("%s/%s",referenceSource, refSourceName) ); + reference.setReferenceSource( String.format("%s/%s", referenceBaseNameSource, refSourceName) ); reference.setReferenceID(uuid.toString()); refs.add(reference); } + } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 9a4c226e9..883cbc1bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPILocation; +import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -32,10 +33,7 @@ import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.BrAPILocationDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; @@ -69,7 +67,8 @@ public class ExperimentProcessor implements Processor { private BrAPILocationDAO brAPILocationDAO; private BrAPIStudyDAO brAPIStudyDAO; private BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private final BrAPIGermplasmDAO brAPIGermplasmDAO; + private BrAPISeasonDAO brAPISeasonDAO; + private BrAPIGermplasmDAO brAPIGermplasmDAO; //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. @@ -88,12 +87,14 @@ public ExperimentProcessor(DSLContext dsl, BrAPILocationDAO brAPILocationDAO, BrAPIStudyDAO brAPIStudyDAO, BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPISeasonDAO brAPISeasonDAO, BrAPIGermplasmDAO brAPIGermplasmDAO) { this.dsl = dsl; this.brapiTrialDAO = brapiTrialDAO; this.brAPILocationDAO = brAPILocationDAO; this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPISeasonDAO = brAPISeasonDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; } @@ -147,11 +148,11 @@ public Map process( mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( importRow.getExpUnitId() ) ); - PendingImportObject germplasmPIO = getGidPio(importRow, mappedImportRow); + PendingImportObject germplasmPIO = getGidPio(importRow); mappedImportRow.setGermplasm( germplasmPIO ); if (! StringUtils.isBlank( importRow.getGid() )) { // if GID is blank, don't bother to check if it is valid. - validateGermplam(importRow,validationErrors, i, germplasmPIO); + validateGermplasm(importRow,validationErrors, i, germplasmPIO); } // Construct Observations -- Done in another card mappedBrAPIImport.put(i, mappedImportRow); @@ -170,14 +171,6 @@ public Map process( private void getNewBrapiData(List importRows, Program program, boolean commit) { - //TODO retrieve sequanceName from the program table. - String studySequenceName = program.getStudySequence(); - if (studySequenceName == null) { - log.error(String.format("Program, %s, is missing a value in the study sequence column.", program.getName())); - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for study import"); - } - Supplier studyNextVal = () -> dsl.nextval(studySequenceName.toLowerCase()); - String opsUnitSequenceName = program.getObsUnitSequence(); if (opsUnitSequenceName == null) { log.error(String.format("Program, %s, is missing a value in the obsUnit sequence column.", program.getName())); @@ -185,20 +178,20 @@ private void getNewBrapiData(List importRows, Program program, bool } Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); - this.trialByNameNoScope.put( importRow.getExpTitle(), trialPIO ); + this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); PendingImportObject locationPIO = createLocationPIO(importRow); - this.locationByName.put( importRow.getEnvLocation(), locationPIO ); + this.locationByName.put(importRow.getEnvLocation(), locationPIO); - PendingImportObject studyPIO = createStudyPIO(program, commit, studyNextVal, importRow); - this.studyByNameNoScope.put( importRow.getEnv(),studyPIO ); + PendingImportObject studyPIO = createStudyPIO(program, commit, importRow); + this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - this.observationUnitByNameNoScope.put( importRow.getExpUnitId(), obsUnitPIO ); + this.observationUnitByNameNoScope.put(importRow.getExpUnitId(), obsUnitPIO); } } @@ -269,10 +262,10 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in } } - private void validateRequiredCell(String value, String columHeader, String errorMessage, ValidationErrors validationErrors, int i) { + private void validateRequiredCell(String value, String columnHeader, String errorMessage, ValidationErrors validationErrors, int i) { if ( StringUtils.isBlank( value )) { addRowError( - columHeader, + columnHeader, errorMessage, validationErrors, i ) ; @@ -293,8 +286,8 @@ private Map getStatisticsMap(List HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's HashSet gidCounter = new HashSet<>(); // set of unique GID's - for (int i = 0; i < importRows.size(); i++) { - ExperimentObservation importRow = (ExperimentObservation) importRows.get(i); + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); @@ -317,7 +310,7 @@ private Map getStatisticsMap(List ); } - private void validateGermplam(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { + private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { // error if GID is not blank but GID dose not already exist if( !StringUtils.isBlank( importRow.getGid()) && germplasmPIO == null ) { addRowError( @@ -328,7 +321,7 @@ private void validateGermplam(ExperimentObservation importRow, ValidationErrors } } - private PendingImportObject getGidPio(ExperimentObservation importRow, PendingImport mappedImportRow) { + private PendingImportObject getGidPio(ExperimentObservation importRow) { if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ return existingGermplasmByGID.get(importRow.getGid()); } @@ -358,7 +351,7 @@ private PendingImportObject createObsUnitPIO(Program progr return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, Supplier studyNextVal, ExperimentObservation importRow ) { + private PendingImportObject createStudyPIO(Program program, boolean commit, ExperimentObservation importRow) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { pio = studyByNameNoScope.get( importRow.getEnv() ) ; @@ -367,7 +360,13 @@ private PendingImportObject createStudyPIO(Program program, boolean PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, studyNextVal, commit, BRAPI_REFERENCE_SOURCE, trialID, id); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, trialID, id); + + if( commit) { + String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); + newStudy.setSeasons(Arrays.asList(seasonID)); + } + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); } return pio; @@ -406,14 +405,15 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { - List trials = ProcessorData.getNewObjects(this.trialByNameNoScope); - List locations = ProcessorData.getNewObjects(this.locationByName); - List studies = ProcessorData.getNewObjects(this.studyByNameNoScope); - List observationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List newLocations = ProcessorData.getNewObjects(this.locationByName); + List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); try { - List createdTrials = new ArrayList<>(); - createdTrials.addAll( brapiTrialDAO.createBrAPITrial(trials, program.getId(), upload) ); + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrial(newTrials, program.getId(), upload)); + + // set the DbId to the for each newly created trial for ( BrAPITrial createdTrial: createdTrials ) { String createdTrialName = createdTrial.getTrialName(); String createdTrialName_no_key = Utilities.removeProgramKey( createdTrialName, program.getKey() ); @@ -422,11 +422,13 @@ public void postBrapiData(Map mappedBrAPIImport, Program String dbid = createdTrial.getTrialDbId(); listedTrial.setTrialDbId( dbid ); } - brAPILocationDAO.createBrAPILocation(locations, program.getId(), upload); + brAPILocationDAO.createBrAPILocation(newLocations, program.getId(), upload); updateStudyDependencyValues(mappedBrAPIImport,program.getKey()); List createdStudies = new ArrayList<>(); - createdStudies.addAll( brAPIStudyDAO.createBrAPIStudy(studies, program.getId(), upload) ); + createdStudies.addAll( brAPIStudyDAO.createBrAPIStudy(newStudies, program.getId(), upload) ); + + // set the DbId to the for each newly created study for( BrAPIStudy createdStudy : createdStudies){ String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData( createdStudy.getStudyName(), program.getKey() ); PendingImportObject pi = this.studyByNameNoScope.get( createdStudy_name_no_key ); @@ -434,34 +436,16 @@ public void postBrapiData(Map mappedBrAPIImport, Program brAPIStudy.setStudyDbId( createdStudy.getStudyDbId() ); } - updateObsUnitDependencyValues(mappedBrAPIImport,program.getKey()); - brAPIObservationUnitDAO.createBrAPIObservationUnits(observationUnits, program.getId(), upload); + updateObsUnitDependencyValues(program.getKey()); + brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); } catch (ApiException e) { - log.error(e.getResponseBody().toString()); + log.error(e.getResponseBody()); throw new InternalServerException(e.toString(), e); } -// // Update our records -// createdTrials.forEach(trial -> { -// PendingImportObject preview = this.trialByNameNoScope.get( Utilities.removeProgramKey( trial.getTrialName(),program.getKey() ) ); -// preview.setBrAPIObject(trial); -// }); -// createdLocations.forEach(location -> { -// PendingImportObject preview = locationByName.get(location.getLocationName()); -// preview.setBrAPIObject(location); -// }); -// createdStudies.forEach(study -> { -// String studyName = Utilities.removeProgramKey(study.getStudyName(),program.getKey() ); -// PendingImportObject preview = this.studyByNameNoScope.get( studyName ) ; -// preview.setBrAPIObject(study); -// }); } -// private void updateDependencyValues(Map mappedBrAPIImport) { -// updateStudyDependencyValues(mappedBrAPIImport); -// } - - private void updateObsUnitDependencyValues(Map mappedBrAPIImport, String programKey) { + private void updateObsUnitDependencyValues(String programKey) { // update study DbIds this.studyByNameNoScope.values().stream() @@ -484,13 +468,6 @@ private void updateStudyDbId(BrAPIStudy study, String programKey) { .forEach(obsUnit -> obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId())); } -// private void updateGermplasmDbId(BrAPIGermplasm germplasm) { -// this.observationUnitByNameNoScope.values().stream() -// .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && -// obsUnit.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) -// .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); -// } - private void updateGermplasmDbId(BrAPIGermplasm germplasm) { this.observationUnitByNameNoScope.values().stream() .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && @@ -554,16 +531,14 @@ private ArrayList getGermplasmByAccessionNumber( private Map> initialize_existingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); List uniqueGermplasmGIDs = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getGid()) + .map(ExperimentObservation::getGid) .distinct() .collect(Collectors.toList()); List existingGermplasms; try { existingGermplasms = this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId()); - existingGermplasms.forEach(existingGermplasm -> { - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); - }); + existingGermplasms.forEach(existingGermplasm -> existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm))); return existingGermplasmByGID; } catch (ApiException e) { // We shouldn't get an error back from our services. If we do, nothing the user can do about it @@ -574,9 +549,9 @@ private Map> initialize_observ Map> observationUnitByNameNoScope = new HashMap<>(); List uniqueObservationUnitNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getExpUnitId()) + .map(ExperimentObservation::getExpUnitId) .distinct() - .filter(name -> null !=name) + .filter(Objects::nonNull) .collect(Collectors.toList()); // check for existing observation units. Don't want to update existing, just create new. @@ -601,9 +576,9 @@ private Map> initialize_observ private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); List uniqueStudyNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getEnv()) + .map(ExperimentObservation::getEnv) .distinct() - .filter(name -> null !=name) + .filter(Objects::nonNull) .collect(Collectors.toList()); List existingStudies; @@ -628,9 +603,9 @@ private Map> initialize_studyByNameNoSco private Map> initialize_uniqueLocationNames(Program program, List experimentImportRows) { Map> locationByName = new HashMap<>(); List uniqueLocationNames = experimentImportRows.stream() - .map(experimentImport -> experimentImport.getEnvLocation()) + .map(ExperimentObservation::getEnvLocation) .distinct() - .filter(name -> null !=name) + .filter(Objects::nonNull) .collect(Collectors.toList()); List existingLocations; @@ -652,16 +627,17 @@ private Map> initialize_trialByNameNoSco List uniqueTrialNames = experimentImportRows.stream() .map(experimentImport -> Utilities.appendProgramKey( experimentImport.getExpTitle(), programKey, null) ) .distinct(). - filter(name -> null !=name) + filter(Objects::nonNull) .collect(Collectors.toList()); List existingTrials; -log.info(uniqueTrialNames.toString()); try { existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); existingTrials.forEach(existingTrial -> { + String simpleTrialName = Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()); + existingTrial.setTrialName(simpleTrialName); trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + simpleTrialName, new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); }); return trialByNameNoScope; @@ -674,5 +650,39 @@ private String simpleStudyName(String scopedName){ return scopedName.replaceFirst(" \\[.*\\]", ""); } + /** + * NOTE: This assumes that the only Season records of interest are ones + * with a blank name or a name that is the same as the year. + * + * @param years this only looks at the first year of the list. + * @param programId the program ID. + * @return the DbId of the season-record associated with the first year + * of the 'years' list (see NOTE above) + */ + private String yearsToSeasonDbId(List years, UUID programId) { + BrAPISeason targetSeason = null; + String year = years.get(0); + List seasons; + try { + seasons = this.brAPISeasonDAO.getSeasonByYear(year, programId); + for( BrAPISeason season : seasons){ + if(null == season.getSeasonName() || season.getSeasonName().isBlank() || year.equals(season.getSeasonName())){ + targetSeason = season; + break; + } + } + if (targetSeason == null){ + BrAPISeason newSeason = new BrAPISeason(); + newSeason.setYear( Integer.parseInt(year) ); + newSeason.setSeasonName( year ); + targetSeason = this.brAPISeasonDAO.addOneSeason( newSeason, programId ); + } + + } catch (ApiException e) { + log.error(e.getResponseBody(), e);; + } + String seasonDbId = (targetSeason==null) ? null : targetSeason.getSeasonDbId(); + return seasonDbId; + } } \ No newline at end of file From ab9cad4c66214b0f979771a8f65a48fc4231d9de Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 09:47:44 -0400 Subject: [PATCH 14/24] [BI-1189] Add experiment sequence number to study name --- .../ExperimentObservation.java | 4 ++-- .../processors/ExperimentProcessor.java | 17 ++++++++++++++--- .../java/org/breedinginsight/model/Program.java | 3 ++- .../services/ProgramService.java | 16 +++++++++++----- ... => V0.5.38__create_experiment_sequence.sql} | 6 +++--- 5 files changed, 32 insertions(+), 14 deletions(-) rename src/main/resources/db/migration/{V0.5.36__create_study_sequence.sql => V0.5.38__create_experiment_sequence.sql} (80%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index b9ce5f441..41312e016 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -142,11 +142,11 @@ public BrAPIStudy constructBrAPIStudy( Program program, boolean commit, String referenceSource, - UUID trialId, + String expSequenceValue, UUID trialId, UUID id) { BrAPIStudy study = new BrAPIStudy(); if ( commit ){ - study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), trialId.toString())); + study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), expSequenceValue)); // Set external reference study.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialId, id,null)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 883cbc1bf..59591aca5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -178,6 +178,17 @@ private void getNewBrapiData(List importRows, Program program, bool } Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + // It is assumed, at this point, that there is only one experiment per import-file + String expSeqValue = null; + if (commit) { + String expUnitSequenceName = program.getExpSequence(); + if (expUnitSequenceName == null) { + log.error(String.format("Program, %s, is missing a value in the exp sequence column.", program.getName())); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for experiment import"); + } + expSeqValue = dsl.nextval(expUnitSequenceName.toLowerCase()).toString(); + } + for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; @@ -187,7 +198,7 @@ private void getNewBrapiData(List importRows, Program program, bool PendingImportObject locationPIO = createLocationPIO(importRow); this.locationByName.put(importRow.getEnvLocation(), locationPIO); - PendingImportObject studyPIO = createStudyPIO(program, commit, importRow); + PendingImportObject studyPIO = createStudyPIO(program, commit, expSeqValue, importRow); this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); @@ -351,7 +362,7 @@ private PendingImportObject createObsUnitPIO(Program progr return pio; } - private PendingImportObject createStudyPIO(Program program, boolean commit, ExperimentObservation importRow) { + private PendingImportObject createStudyPIO(Program program, boolean commit, String expSeqenceValue, ExperimentObservation importRow) { PendingImportObject pio = null; if( studyByNameNoScope.containsKey( importRow.getEnv()) ) { pio = studyByNameNoScope.get( importRow.getEnv() ) ; @@ -360,7 +371,7 @@ private PendingImportObject createStudyPIO(Program program, boolean PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, trialID, id); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSeqenceValue, trialID, id); if( commit) { String seasonID = this.yearsToSeasonDbId(newStudy.getSeasons(), program.getId()); diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index e69cc6f8f..1c088f10d 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -86,7 +86,8 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .updatedBy(record.getValue(programTable.UPDATED_BY)) .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) - .studySequence(record.getValue(programTable.STUDY_SEQUENCE)) + // .studySequence(record.getValue(programTable.STUDY_SEQUENCE)) + .expSequence( record.getValue(programTable.EXP_SEQUENCE)) .obsUnitSequence(( record.getValue(programTable.OBS_UNIT_SEQUENCE))) .build(); diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index 1ac0357cc..b89487b90 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -59,8 +59,9 @@ public class ProgramService { private static final String PROGRAM_NAME_IN_USE = "PROGRAM_NAME_IN_USE"; private static final String PROGRAM_KEY_IN_USE = "PROGRAM_KEY_IN_USE"; private static final String GERMPLASM_SEQUENCE_TEMPLATE = "%s_germplasm_sequence"; - private static final String STUDY_SEQUENCE_TEMPLATE = "%s_study_sequence"; +// private static final String STUDY_SEQUENCE_TEMPLATE = "%s_study_sequence"; private static final String OBS_UNIT_SEQUENCE_TEMPLATE = "%s_obs_unit_sequence"; + private static final String EXP_SEQUENCE_TEMPLATE = "%s_exp_sequence"; @Inject public ProgramService(ProgramDAO dao, ProgramOntologyDAO programOntologyDAO, ProgramObservationLevelDAO programObservationLevelDAO, @@ -163,9 +164,13 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse String germplasm_sequence_name = String.format(GERMPLASM_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(germplasm_sequence_name).execute(); - // Create study sequence - String study_sequence_name = String.format(STUDY_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); - dsl.createSequence(study_sequence_name).execute(); +// // Create study sequence +// String study_sequence_name = String.format(STUDY_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); +// dsl.createSequence(study_sequence_name).execute(); + + // Create experiment sequence + String exp_sequence_name = String.format(EXP_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); + dsl.createSequence(exp_sequence_name).execute(); // Create obs unit sequence String obs_unit_sequence_name = String.format(OBS_UNIT_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); @@ -181,7 +186,8 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .brapiUrl(brapiUrl) .key(programRequest.getKey()) .germplasmSequence(germplasm_sequence_name) - .studySequence(study_sequence_name) + //.studySequence(study_sequence_name) + .expSequence( exp_sequence_name ) .obsUnitSequence( obs_unit_sequence_name ) .createdBy(actingUser.getId()) .updatedBy(actingUser.getId()) diff --git a/src/main/resources/db/migration/V0.5.36__create_study_sequence.sql b/src/main/resources/db/migration/V0.5.38__create_experiment_sequence.sql similarity index 80% rename from src/main/resources/db/migration/V0.5.36__create_study_sequence.sql rename to src/main/resources/db/migration/V0.5.38__create_experiment_sequence.sql index 04c2f11b4..1d2b97e83 100644 --- a/src/main/resources/db/migration/V0.5.36__create_study_sequence.sql +++ b/src/main/resources/db/migration/V0.5.38__create_experiment_sequence.sql @@ -15,7 +15,7 @@ * limitations under the License. */ -alter table program add column study_sequence text; +alter table program add column exp_sequence text; do $$ @@ -27,8 +27,8 @@ loop if f.key is NULL then RAISE EXCEPTION 'Programs must have a key associated with them'; end if; - execute format('create sequence %s_study_sequence',f.key); - update program set study_sequence = format('%s_study_sequence', f.key) where id = f.id; + execute format('create sequence %s_exp_sequence',f.key); + update program set exp_sequence = format('%s_exp_sequence', f.key) where id = f.id; end loop; end; $$ \ No newline at end of file From 4bf78f1ef01a127eba2aa3a445a1f89522e9eafc Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 10:24:18 -0400 Subject: [PATCH 15/24] [BI-1189] remove references to study sequence --- .../experimentObservation/ExperimentObservation.java | 3 ++- src/main/java/org/breedinginsight/model/Program.java | 1 - .../java/org/breedinginsight/services/ProgramService.java | 6 ------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 41312e016..e94c5728d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -142,7 +142,8 @@ public BrAPIStudy constructBrAPIStudy( Program program, boolean commit, String referenceSource, - String expSequenceValue, UUID trialId, + String expSequenceValue, + UUID trialId, UUID id) { BrAPIStudy study = new BrAPIStudy(); if ( commit ){ diff --git a/src/main/java/org/breedinginsight/model/Program.java b/src/main/java/org/breedinginsight/model/Program.java index 1c088f10d..26a7a4ab6 100644 --- a/src/main/java/org/breedinginsight/model/Program.java +++ b/src/main/java/org/breedinginsight/model/Program.java @@ -86,7 +86,6 @@ public static Program parseSQLRecord(Record record, ProgramTable programTable) { .updatedBy(record.getValue(programTable.UPDATED_BY)) .active(record.getValue(programTable.ACTIVE)) .germplasmSequence(record.getValue(programTable.GERMPLASM_SEQUENCE)) - // .studySequence(record.getValue(programTable.STUDY_SEQUENCE)) .expSequence( record.getValue(programTable.EXP_SEQUENCE)) .obsUnitSequence(( record.getValue(programTable.OBS_UNIT_SEQUENCE))) .build(); diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index b89487b90..70f218049 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -59,7 +59,6 @@ public class ProgramService { private static final String PROGRAM_NAME_IN_USE = "PROGRAM_NAME_IN_USE"; private static final String PROGRAM_KEY_IN_USE = "PROGRAM_KEY_IN_USE"; private static final String GERMPLASM_SEQUENCE_TEMPLATE = "%s_germplasm_sequence"; -// private static final String STUDY_SEQUENCE_TEMPLATE = "%s_study_sequence"; private static final String OBS_UNIT_SEQUENCE_TEMPLATE = "%s_obs_unit_sequence"; private static final String EXP_SEQUENCE_TEMPLATE = "%s_exp_sequence"; @@ -164,10 +163,6 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse String germplasm_sequence_name = String.format(GERMPLASM_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(germplasm_sequence_name).execute(); -// // Create study sequence -// String study_sequence_name = String.format(STUDY_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); -// dsl.createSequence(study_sequence_name).execute(); - // Create experiment sequence String exp_sequence_name = String.format(EXP_SEQUENCE_TEMPLATE, programRequest.getKey()).toLowerCase(); dsl.createSequence(exp_sequence_name).execute(); @@ -186,7 +181,6 @@ public Program create(ProgramRequest programRequest, AuthenticatedUser actingUse .brapiUrl(brapiUrl) .key(programRequest.getKey()) .germplasmSequence(germplasm_sequence_name) - //.studySequence(study_sequence_name) .expSequence( exp_sequence_name ) .obsUnitSequence( obs_unit_sequence_name ) .createdBy(actingUser.getId()) From b87056a152dc10fdeeb447b63124f925dc83a49d Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 May 2022 15:39:27 -0400 Subject: [PATCH 16/24] [BI-1189] caching for year-to-seasonDbId --- .../processors/ExperimentProcessor.java | 112 ++++++++++++------ 1 file changed, 74 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 59591aca5..15278c1dc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -70,14 +70,17 @@ public class ExperimentProcessor implements Processor { private BrAPISeasonDAO brAPISeasonDAO; private BrAPIGermplasmDAO brAPIGermplasmDAO; + // used to make the yearsToSeasonDbId() function more efficient + private Map yearToSeasonDbIdCache = new HashMap<>(); + //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; // It is assumed that there are no pre-existing observation units for the given environment (so this will not be - // intitalized by getExistingBrapiData - private Map> observationUnitByNameNoScope = null; + // initialized by getExistingBrapiData() ) + private Map> observationUnitByNameNoScope = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -112,7 +115,7 @@ public void getExistingBrapiData(List importRows, Program program) this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); this.locationByName = initialize_uniqueLocationNames ( program, experimentImportRows ); this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); - this.observationUnitByNameNoScope = initialize_observationUnitByNameNoScope( program, experimentImportRows ); +// this.observationUnitByNameNoScope = initialize_observationUnitByNameNoScope( program, experimentImportRows ); this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); } @@ -147,7 +150,7 @@ public Map process( mappedImportRow.setTrial( this.trialByNameNoScope.get( importRow.getExpTitle() ) ); mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); - mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( importRow.getExpUnitId() ) ); + mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createOUKey( importRow ) ) ); PendingImportObject germplasmPIO = getGidPio(importRow); mappedImportRow.setGermplasm( germplasmPIO ); @@ -202,10 +205,16 @@ private void getNewBrapiData(List importRows, Program program, bool this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - this.observationUnitByNameNoScope.put(importRow.getExpUnitId(), obsUnitPIO); + String key = createOUKey(importRow); + this.observationUnitByNameNoScope.put(key, obsUnitPIO); } } + private String createOUKey(ExperimentObservation importRow) { + String key = importRow.getEnv() + importRow.getExpUnitId(); + return key; + } + private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors) { HashSet uniqueStudyAndObsUnit = new HashSet<>(); for (int i = 0; i < importRows.size(); i++) { @@ -216,8 +225,21 @@ private ValidationErrors validateFields(List importRows, Validation return validationErrors; } - private void validateUniqueObsUnits(ValidationErrors validationErrors, HashSet uniqueStudyAndObsUnit, int i, ExperimentObservation importRow) { - String envIdPlusStudyId = importRow.getEnv() + importRow.getExpUnitId(); + /** + * Validate that the the observation unit is unique within a study. + *
+ * SIDE EFFECTS: validationErrors and uniqueStudyAndObsUnit can be modified. + * @param validationErrors can be modified as a side effect. + * @param uniqueStudyAndObsUnit can be modified as a side effect. + * @param i counter that is always two less the file row being validated + * @param importRow the data row being validated + */ + private void validateUniqueObsUnits( + ValidationErrors validationErrors, + HashSet uniqueStudyAndObsUnit, + int i, + ExperimentObservation importRow) { + String envIdPlusStudyId = createOUKey( importRow ); if( uniqueStudyAndObsUnit.contains( envIdPlusStudyId )){ String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); @@ -301,7 +323,7 @@ private Map getStatisticsMap(List ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, importRow.getExpUnitId()); + addIfNotNull(obsUnitsIDCounter, createOUKey( importRow )); addIfNotNull(gidCounter, importRow.getGid()); } ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() @@ -343,8 +365,8 @@ private PendingImportObject getGidPio(ExperimentObservation impo private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { PendingImportObject pio = null; - if( this.observationUnitByNameNoScope.containsKey( importRow.getExpUnitId() ) ) { - pio = observationUnitByNameNoScope.get(importRow.getExpUnitId() ) ; + if( this.observationUnitByNameNoScope.containsKey( createOUKey( importRow ) ) ) { + pio = observationUnitByNameNoScope.get( createOUKey( importRow ) ) ; } else{ String germplasmName = ""; @@ -556,33 +578,33 @@ private Map> initialize_existingGerm throw new InternalServerException(e.toString(), e); } } - private Map> initialize_observationUnitByNameNoScope(Program program, List experimentImportRows) { - - Map> observationUnitByNameNoScope = new HashMap<>(); - List uniqueObservationUnitNames = experimentImportRows.stream() - .map(ExperimentObservation::getExpUnitId) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - // check for existing observation units. Don't want to update existing, just create new. - // TODO: do we allow adding observations to existing studies? yes, but not updating - // ignore all data for observation units existing in system - - List existingObservationUnits; - - try { - existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); - existingObservationUnits.forEach(existingObservationUnit -> { - // update mapped brapi import, does in process - observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); - }); - return observationUnitByNameNoScope; - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - } +// private Map> initialize_observationUnitByNameNoScope(Program program, List experimentImportRows) { +// +// Map> observationUnitByNameNoScope = new HashMap<>(); +// List uniqueObservationUnitNames = experimentImportRows.stream() +// .map(ExperimentObservation::getExpUnitId) +// .distinct() +// .filter(Objects::nonNull) +// .collect(Collectors.toList()); +// +// // check for existing observation units. Don't want to update existing, just create new. +// // TODO: do we allow adding observations to existing studies? yes, but not updating +// // ignore all data for observation units existing in system +// +// List existingObservationUnits; +// +// try { +// existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); +// existingObservationUnits.forEach(existingObservationUnit -> { +// // update mapped brapi import, does in process +// observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); +// }); +// return observationUnitByNameNoScope; +// } catch (ApiException e) { +// // We shouldn't get an error back from our services. If we do, nothing the user can do about it +// throw new InternalServerException(e.toString(), e); +// } +// } private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); @@ -662,6 +684,8 @@ private String simpleStudyName(String scopedName){ } /** + * Converts year String to SeasonDbId + *
* NOTE: This assumes that the only Season records of interest are ones * with a blank name or a name that is the same as the year. * @@ -671,8 +695,20 @@ private String simpleStudyName(String scopedName){ * of the 'years' list (see NOTE above) */ private String yearsToSeasonDbId(List years, UUID programId) { - BrAPISeason targetSeason = null; String year = years.get(0); + String dbID = null; + if (this.yearToSeasonDbIdCache.containsKey(year) ){ // get it from cache if possable + dbID = this.yearToSeasonDbIdCache.get(year); + } + else{ + dbID = this.yearToSeasonDbIdFromDatabase(year,programId); + this.yearToSeasonDbIdCache.put(year, dbID); + } + return dbID; + } + + private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { + BrAPISeason targetSeason = null; List seasons; try { seasons = this.brAPISeasonDAO.getSeasonByYear(year, programId); From c56f5a72c7d7668d2a4f6b7ddeae22f53c426850 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 2 Jun 2022 13:16:08 -0400 Subject: [PATCH 17/24] [BI-1189]Addrssed the PR comments by HMS17 --- .../ExperimentObservation.java | 7 ++--- .../processors/ExperimentProcessor.java | 26 +++++++++---------- ... => V0.5.36__create_obs_unit_sequence.sql} | 0 ...> V0.5.37__create_experiment_sequence.sql} | 0 4 files changed, 15 insertions(+), 18 deletions(-) rename src/main/resources/db/migration/{V0.5.37__create_obs_unit_sequence.sql => V0.5.36__create_obs_unit_sequence.sql} (100%) rename src/main/resources/db/migration/{V0.5.38__create_experiment_sequence.sql => V0.5.37__create_experiment_sequence.sql} (100%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index e94c5728d..55cca9e92 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -75,8 +75,7 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id="envLocation", name="Environment Location", description = "Location of the environment") private String envLocation; - //TODO make this ImportFieldTypeEnum.INTEGER - @ImportFieldType(type= ImportFieldTypeEnum.TEXT) + @ImportFieldType(type= ImportFieldTypeEnum.INTEGER) @ImportFieldMetadata(id="envYear", name="Environment Year", description = "Year corresponding to the environment") private String envYear; @@ -125,7 +124,6 @@ public BrAPITrial constructBrAPITrial(Program program, boolean commit, String re trial.setProgramDbId(brapiProgram.getProgramDbId()); trial.setProgramName(brapiProgram.getProgramName()); - // TODO: Get to a constant trial.putAdditionalInfoItem("defaultObservationLevel", getExpUnit()); trial.putAdditionalInfoItem("experimentType", getExpType()); @@ -193,7 +191,6 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( } observationUnit.setStudyName(getEnv()); - // TODO: Set the germplasm if(germplasmName==null){ germplasmName = getGermplasmName(); } @@ -201,7 +198,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName("plot"); + level.setLevelName("plot"); //BreedBase only excepts "plot" or "plant" level.setLevelCode( getExpUnitId() ); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem("observationLevel", getExpUnit()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 15278c1dc..779748452 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -113,7 +113,7 @@ public void getExistingBrapiData(List importRows, Program program) .collect(Collectors.toList()); this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); - this.locationByName = initialize_uniqueLocationNames ( program, experimentImportRows ); + this.locationByName = initialize_uniqueLocationNames( program, experimentImportRows ); this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); // this.observationUnitByNameNoScope = initialize_observationUnitByNameNoScope( program, experimentImportRows ); this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); @@ -121,7 +121,7 @@ public void getExistingBrapiData(List importRows, Program program) /** * @param importRows - one element of the list for every row of the import file. - * @param mappedBrAPIImport - passed in by reference and modified within this program (this will latter be passed to the front end for the preview) + * @param mappedBrAPIImport - passed in by reference and modified within this program (this will later be passed to the front end for the preview) * @param program * @param user * @param commit - true when the data should be saved (ie when the user has pressed the "Commit" button) @@ -150,7 +150,7 @@ public Map process( mappedImportRow.setTrial( this.trialByNameNoScope.get( importRow.getExpTitle() ) ); mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); - mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createOUKey( importRow ) ) ); + mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ); PendingImportObject germplasmPIO = getGidPio(importRow); mappedImportRow.setGermplasm( germplasmPIO ); @@ -174,12 +174,12 @@ public Map process( private void getNewBrapiData(List importRows, Program program, boolean commit) { - String opsUnitSequenceName = program.getObsUnitSequence(); - if (opsUnitSequenceName == null) { + String obsUnitSequenceName = program.getObsUnitSequence(); + if (obsUnitSequenceName == null) { log.error(String.format("Program, %s, is missing a value in the obsUnit sequence column.", program.getName())); throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for observation unit import"); } - Supplier obsUnitNextVal = () -> dsl.nextval(opsUnitSequenceName.toLowerCase()); + Supplier obsUnitNextVal = () -> dsl.nextval(obsUnitSequenceName.toLowerCase()); // It is assumed, at this point, that there is only one experiment per import-file String expSeqValue = null; @@ -205,12 +205,12 @@ private void getNewBrapiData(List importRows, Program program, bool this.studyByNameNoScope.put(importRow.getEnv(), studyPIO); PendingImportObject obsUnitPIO = createObsUnitPIO(program, commit, obsUnitNextVal, importRow); - String key = createOUKey(importRow); + String key = createObservationUnitKey(importRow); this.observationUnitByNameNoScope.put(key, obsUnitPIO); } } - private String createOUKey(ExperimentObservation importRow) { + private String createObservationUnitKey(ExperimentObservation importRow) { String key = importRow.getEnv() + importRow.getExpUnitId(); return key; } @@ -239,7 +239,7 @@ private void validateUniqueObsUnits( HashSet uniqueStudyAndObsUnit, int i, ExperimentObservation importRow) { - String envIdPlusStudyId = createOUKey( importRow ); + String envIdPlusStudyId = createObservationUnitKey( importRow ); if( uniqueStudyAndObsUnit.contains( envIdPlusStudyId )){ String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); this.addRowError("Exp Unit ID", errorMessage, validationErrors, i); @@ -323,7 +323,7 @@ private Map getStatisticsMap(List ExperimentObservation importRow = (ExperimentObservation) row; // Collect date for stats. addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, createOUKey( importRow )); + addIfNotNull(obsUnitsIDCounter, createObservationUnitKey( importRow )); addIfNotNull(gidCounter, importRow.getGid()); } ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() @@ -344,7 +344,7 @@ private Map getStatisticsMap(List } private void validateGermplasm(ExperimentObservation importRow, ValidationErrors validationErrors, int i, PendingImportObject germplasmPIO) { - // error if GID is not blank but GID dose not already exist + // error if GID is not blank but GID does not already exist if( !StringUtils.isBlank( importRow.getGid()) && germplasmPIO == null ) { addRowError( "GID", @@ -365,8 +365,8 @@ private PendingImportObject getGidPio(ExperimentObservation impo private PendingImportObject createObsUnitPIO(Program program, boolean commit, Supplier obsUnitNextVal, ExperimentObservation importRow) { PendingImportObject pio = null; - if( this.observationUnitByNameNoScope.containsKey( createOUKey( importRow ) ) ) { - pio = observationUnitByNameNoScope.get( createOUKey( importRow ) ) ; + if( this.observationUnitByNameNoScope.containsKey( createObservationUnitKey( importRow ) ) ) { + pio = observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ; } else{ String germplasmName = ""; diff --git a/src/main/resources/db/migration/V0.5.37__create_obs_unit_sequence.sql b/src/main/resources/db/migration/V0.5.36__create_obs_unit_sequence.sql similarity index 100% rename from src/main/resources/db/migration/V0.5.37__create_obs_unit_sequence.sql rename to src/main/resources/db/migration/V0.5.36__create_obs_unit_sequence.sql diff --git a/src/main/resources/db/migration/V0.5.38__create_experiment_sequence.sql b/src/main/resources/db/migration/V0.5.37__create_experiment_sequence.sql similarity index 100% rename from src/main/resources/db/migration/V0.5.38__create_experiment_sequence.sql rename to src/main/resources/db/migration/V0.5.37__create_experiment_sequence.sql From 4d6ac34ecee4596d6b767ca4152bf8b6742672e7 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 7 Jun 2022 10:00:19 -0400 Subject: [PATCH 18/24] [BI-1189]Addrssed more PR comments by HMS17 --- .../ExperimentObservation.java | 2 +- .../processors/ExperimentProcessor.java | 39 +++---------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 55cca9e92..fadcccfea 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -198,7 +198,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName("plot"); //BreedBase only excepts "plot" or "plant" + level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" level.setLevelCode( getExpUnitId() ); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem("observationLevel", getExpUnit()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 779748452..d55ecedc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -78,9 +78,9 @@ public class ExperimentProcessor implements Processor { private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; - // It is assumed that there are no pre-existing observation units for the given environment (so this will not be + // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) - private Map> observationUnitByNameNoScope = new HashMap<>(); + private Map> observationUnitByNameNoScope = null; // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -115,7 +115,8 @@ public void getExistingBrapiData(List importRows, Program program) this.trialByNameNoScope = initialize_trialByNameNoScope( program, experimentImportRows ); this.locationByName = initialize_uniqueLocationNames( program, experimentImportRows ); this.studyByNameNoScope = initialize_studyByNameNoScope( program, experimentImportRows ); -// this.observationUnitByNameNoScope = initialize_observationUnitByNameNoScope( program, experimentImportRows ); + // All of the Observation Units will be new. None will be preexisting. + this.observationUnitByNameNoScope = new HashMap<>(); this.existingGermplasmByGID = initialize_existingGermplasmByGID( program, experimentImportRows ); } @@ -151,7 +152,7 @@ public Map process( mappedImportRow.setLocation( this.locationByName.get( importRow.getEnvLocation() ) ); mappedImportRow.setStudy( this.studyByNameNoScope.get( importRow.getEnv() ) ); mappedImportRow.setObservationUnit( this.observationUnitByNameNoScope.get( createObservationUnitKey( importRow ) ) ); - PendingImportObject germplasmPIO = getGidPio(importRow); + PendingImportObject germplasmPIO = getGidPOI(importRow); mappedImportRow.setGermplasm( germplasmPIO ); if (! StringUtils.isBlank( importRow.getGid() )) { // if GID is blank, don't bother to check if it is valid. @@ -354,7 +355,7 @@ private void validateGermplasm(ExperimentObservation importRow, ValidationErrors } } - private PendingImportObject getGidPio(ExperimentObservation importRow) { + private PendingImportObject getGidPOI(ExperimentObservation importRow) { if( this.existingGermplasmByGID.containsKey( importRow.getGid() )){ return existingGermplasmByGID.get(importRow.getGid()); } @@ -543,7 +544,6 @@ public String getName() { return NAME; } - //TODO should this method be moved to BrAPIExperimentService.java private ArrayList getGermplasmByAccessionNumber( List germplasmAccessionNumbers, UUID programId) throws ApiException { @@ -578,33 +578,6 @@ private Map> initialize_existingGerm throw new InternalServerException(e.toString(), e); } } -// private Map> initialize_observationUnitByNameNoScope(Program program, List experimentImportRows) { -// -// Map> observationUnitByNameNoScope = new HashMap<>(); -// List uniqueObservationUnitNames = experimentImportRows.stream() -// .map(ExperimentObservation::getExpUnitId) -// .distinct() -// .filter(Objects::nonNull) -// .collect(Collectors.toList()); -// -// // check for existing observation units. Don't want to update existing, just create new. -// // TODO: do we allow adding observations to existing studies? yes, but not updating -// // ignore all data for observation units existing in system -// -// List existingObservationUnits; -// -// try { -// existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); -// existingObservationUnits.forEach(existingObservationUnit -> { -// // update mapped brapi import, does in process -// observationUnitByNameNoScope.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); -// }); -// return observationUnitByNameNoScope; -// } catch (ApiException e) { -// // We shouldn't get an error back from our services. If we do, nothing the user can do about it -// throw new InternalServerException(e.toString(), e); -// } -// } private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); From 14e5f7b1912d9222acea81a9629124956a9950a6 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 6 Jul 2022 14:12:21 -0400 Subject: [PATCH 19/24] [BI-1189] responses of PR comments from Chris T --- .../constants/BrAPIAdditionalInfoFields.java | 2 ++ .../brapps/importer/daos/BrAPISeasonDAO.java | 2 +- .../ExperimentObservation.java | 29 +++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 13eb6f9b8..028e51bca 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -20,4 +20,6 @@ public final class BrAPIAdditionalInfoFields { public static final String GERMPLASM_RAW_PEDIGREE = "rawPedigree"; public static final String GERMPLASM_PEDIGREE_BY_NAME = "pedigreeByName"; + public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel"; + public static final String EXPERIMENT_TYPE = "experimentType"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java index 2685b0245..235c752fb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java @@ -45,8 +45,8 @@ public List getSeasonByYear(String year, UUID programId) throws Api SeasonQueryParams queryParams = SeasonQueryParams.builder() .year( year ) + .pageSize( 10000 ) .build(); - List seasons = new ArrayList<>(); ApiResponse apiResponse = api.seasonsGet( queryParams ); BrAPISeasonListResponse seasonListResponse = apiResponse.getBody(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index fadcccfea..d512111f6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -23,6 +23,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.model.Program; @@ -114,7 +115,7 @@ public BrAPITrial constructBrAPITrial(Program program, boolean commit, String re trial.setTrialName( Utilities.appendProgramKey(getExpTitle(), program.getKey() )); // Set external reference - trial.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, id, null, null)); + trial.setExternalReferences(getTrialExternalReferences(program, referenceSource, id)); } else{ trial.setTrialName( getExpTitle() ); @@ -124,8 +125,8 @@ public BrAPITrial constructBrAPITrial(Program program, boolean commit, String re trial.setProgramDbId(brapiProgram.getProgramDbId()); trial.setProgramName(brapiProgram.getProgramName()); - trial.putAdditionalInfoItem("defaultObservationLevel", getExpUnit()); - trial.putAdditionalInfoItem("experimentType", getExpType()); + trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); + trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); return trial; } @@ -148,7 +149,7 @@ public BrAPIStudy constructBrAPIStudy( study.setStudyName(Utilities.appendProgramKey(getEnv(), program.getKey(), expSequenceValue)); // Set external reference - study.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialId, id,null)); + study.setExternalReferences(getStudyExternalReferences(program, referenceSource, trialId, id)); } else { study.setStudyName(getEnv()); @@ -159,7 +160,9 @@ public BrAPIStudy constructBrAPIStudy( study.setTrialName(getExpTitle()); study.setSeasons( List.of( getEnvYear()==null ? "" : getEnvYear() ) ); - String designType = "Analysis"; + String designType = "Analysis"; // to support the BRApi server, the design type must be one of the following: + // 'CRD','Alpha','MAD','Lattice','Augmented','RCBD','p-rep','splitplot','greenhouse','Westcott', or 'Analysis' + // For now it will be hardcoded to 'Analysis' BrAPIStudyExperimentalDesign design = new BrAPIStudyExperimentalDesign(); design.setPUI(designType); design.setDescription(designType); @@ -184,7 +187,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( observationUnit.setObservationUnitName( Utilities.appendProgramKey(getExpUnitId(), program.getKey(), nextVal.get().toString())); // Set external reference - observationUnit.setExternalReferences(getBrAPIExternalReferences(program, referenceSource, trialID, studyID, id)); + observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, trialID, studyID, id)); } else { observationUnit.setObservationUnitName(getExpUnitId()); @@ -261,6 +264,20 @@ private List getBrAPIExternalReferences( return refs; } + private List getTrialExternalReferences( + Program program, String referenceSourceBaseName, UUID trialId) { + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, null, null); + } + private List getStudyExternalReferences( + Program program, String referenceSourceBaseName, UUID trialId, UUID studyId) { + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); + } + private List getObsUnitExternalReferences( + Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { + return getBrAPIExternalReferences(program, referenceSourceBaseName, trialId, studyId, null); + } + + private void addReference(List refs, UUID uuid, String referenceBaseNameSource, String refSourceName) { BrAPIExternalReference reference; reference = new BrAPIExternalReference(); From 4ca4dfe1127c4dc498666f6237647f1b16cf751e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 8 Jul 2022 13:02:36 -0400 Subject: [PATCH 20/24] [BI-1189]Addrssed more PR comments by Chris T --- .../v2/constants/BrAPIAdditionalInfoFields.java | 1 + .../experimentObservation/ExperimentObservation.java | 12 ++++++++---- .../services/processors/ExperimentProcessor.java | 2 +- .../org/breedinginsight/model/BrAPIConstants.java | 4 +++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 028e51bca..3327425be 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -21,5 +21,6 @@ public final class BrAPIAdditionalInfoFields { public static final String GERMPLASM_RAW_PEDIGREE = "rawPedigree"; public static final String GERMPLASM_PEDIGREE_BY_NAME = "pedigreeByName"; public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel"; + public static final String OBSERVATION_LEVEL = "observationLevel"; public static final String EXPERIMENT_TYPE = "experimentType"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index d512111f6..5e30bb776 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -26,6 +26,7 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -158,7 +159,10 @@ public BrAPIStudy constructBrAPIStudy( study.setStudyType(getExpType()); study.setLocationName(getEnvLocation()); study.setTrialName(getExpTitle()); - study.setSeasons( List.of( getEnvYear()==null ? "" : getEnvYear() ) ); + + List seasonList = new ArrayList<>(); + seasonList.add( getEnvYear() ); + study.setSeasons( seasonList ); String designType = "Analysis"; // to support the BRApi server, the design type must be one of the following: // 'CRD','Alpha','MAD','Lattice','Augmented','RCBD','p-rep','splitplot','greenhouse','Westcott', or 'Analysis' @@ -204,13 +208,13 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" level.setLevelCode( getExpUnitId() ); position.setObservationLevel(level); - observationUnit.putAdditionalInfoItem("observationLevel", getExpUnit()); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); // Exp Unit List levelRelationships = new ArrayList<>(); if( getExpReplicateNo() !=null ) { BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); - repLvl.setLevelName("replicate"); + repLvl.setLevelName(BrAPIConstants.REPLICATE.getValue()); repLvl.setLevelCode(getExpReplicateNo()); levelRelationships.add(repLvl); } @@ -218,7 +222,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( // Block number if( getExpBlockNo() != null ) { BrAPIObservationUnitLevelRelationship repLvl = new BrAPIObservationUnitLevelRelationship(); - repLvl.setLevelName("block"); + repLvl.setLevelName( BrAPIConstants.REPLICATE.getValue() ); repLvl.setLevelCode(getExpBlockNo()); levelRelationships.add(repLvl); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d55ecedc3..e30b88ee3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -686,7 +686,7 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { try { seasons = this.brAPISeasonDAO.getSeasonByYear(year, programId); for( BrAPISeason season : seasons){ - if(null == season.getSeasonName() || season.getSeasonName().isBlank() || year.equals(season.getSeasonName())){ + if(null == season.getSeasonName() || season.getSeasonName().isBlank() || season.getSeasonName().equals(year)){ targetSeason = season; break; } diff --git a/src/main/java/org/breedinginsight/model/BrAPIConstants.java b/src/main/java/org/breedinginsight/model/BrAPIConstants.java index 47ba71804..03860a4df 100644 --- a/src/main/java/org/breedinginsight/model/BrAPIConstants.java +++ b/src/main/java/org/breedinginsight/model/BrAPIConstants.java @@ -3,7 +3,9 @@ import com.fasterxml.jackson.annotation.JsonValue; public enum BrAPIConstants { - SYSTEM_DEFAULT("System Default"); + SYSTEM_DEFAULT("System Default"), + REPLICATE( "replicate"), + BLOCK( "bolck"); private String value; From 71b5351f382361b1951f2e836eb1b4d866be5cf2 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 20 Jul 2022 11:09:53 -0400 Subject: [PATCH 21/24] [BI-1189] appending experiment number to study names when searching for existing studies --- .../constants/BrAPIAdditionalInfoFields.java | 1 + .../processors/ExperimentProcessor.java | 33 ++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 1b4ad2322..7fe94de1f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -35,4 +35,5 @@ public final class BrAPIAdditionalInfoFields { public static final String DEFAULT_OBSERVATION_LEVEL = "defaultObservationLevel"; public static final String OBSERVATION_LEVEL = "observationLevel"; public static final String EXPERIMENT_TYPE = "experimentType"; + public static final String EXPERIMENT_NUMBER = "experimentNumber"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c4b8ab561..48dac3659 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -32,6 +32,7 @@ import org.brapi.v2.model.pheno.*; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -106,6 +107,7 @@ public ExperimentProcessor(DSLContext dsl, * @param importRows * @param program */ + @Override public void getExistingBrapiData(List importRows, Program program) { List experimentImportRows = importRows.stream() @@ -582,7 +584,15 @@ private Map> initialize_existingGerm private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); List uniqueStudyNames = experimentImportRows.stream() - .map(ExperimentObservation::getEnv) + .map(experimentObservation -> { + String envName = experimentObservation.getEnv(); + PendingImportObject trial = trialByNameNoScope.get(experimentObservation.getExpTitle()); + if(trial.getBrAPIObject().getAdditionalInfo() != null) { + envName = Utilities.appendProgramKey(envName, program.getName(), trial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); + } + + return envName; + }) .distinct() .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -591,12 +601,14 @@ private Map> initialize_studyByNameNoSco try { existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); existingStudies.forEach(existingStudy -> { - String studySequence = null; - if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { - studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); - } + // I don't think this is needed anymore +// String studySequence = null; +// if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { +// studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); +// } + existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); studyByNameNoScope.put( - Utilities.removeProgramKey(existingStudy.getStudyName(), program.getKey(), studySequence), + existingStudy.getStudyName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy)); }); return studyByNameNoScope; @@ -632,18 +644,17 @@ private Map> initialize_trialByNameNoSco String programKey = program.getKey(); List uniqueTrialNames = experimentImportRows.stream() .map(experimentImport -> Utilities.appendProgramKey( experimentImport.getExpTitle(), programKey, null) ) - .distinct(). - filter(Objects::nonNull) + .distinct() + .filter(Objects::nonNull) .collect(Collectors.toList()); List existingTrials; try { existingTrials = brapiTrialDAO.getTrialByName(uniqueTrialNames, program); existingTrials.forEach(existingTrial -> { - String simpleTrialName = Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()); - existingTrial.setTrialName(simpleTrialName); + existingTrial.setTrialName(Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey())); trialByNameNoScope.put( - simpleTrialName, + existingTrial.getTrialName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial)); }); return trialByNameNoScope; From 2bfdf318990559bf461d12cc20181d8684b1dfd9 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Sat, 23 Jul 2022 12:48:39 -0400 Subject: [PATCH 22/24] [BI-1189] added experiment sequence number to study name --- .../brapps/importer/daos/BrAPIStudyDAO.java | 16 ++++++ .../ExperimentObservation.java | 3 +- .../processors/ExperimentProcessor.java | 53 +++++++++++-------- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index ab9197666..3752a394b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.daos; +import io.micronaut.context.annotation.Property; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.core.StudiesApi; import org.brapi.v2.model.core.BrAPIStudy; @@ -32,6 +33,8 @@ @Singleton public class BrAPIStudyDAO { + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; private ProgramDAO programDAO; private ImportDAO importDAO; @@ -54,6 +57,19 @@ public List getStudyByName(List studyNames, Program program) ); } + public List getStudiesByExperimentID(UUID experimentID, Program program ) throws ApiException { + BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); + studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + studySearch.addExternalReferenceIDsItem(experimentID.toString()); + studySearch.addExternalReferenceSourcesItem(BRAPI_REFERENCE_SOURCE + "/trials"); + StudiesApi api = new StudiesApi(programDAO.getCoreClient(program.getId())); + return BrAPIDAOUtil.search( + api::searchStudiesPost, + api::searchStudiesSearchResultsDbIdGet, + studySearch + ); + } + public List createBrAPIStudy(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = new StudiesApi(programDAO.getCoreClient(programId)); return BrAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 5e30bb776..2e05620b2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -109,7 +109,7 @@ public class ExperimentObservation implements BrAPIImport { @ImportFieldMetadata(id="ObsUnitID", name="Observation Unit ID", description = "A database generated unique identifier for experimental observation units") private String ObsUnitID; - public BrAPITrial constructBrAPITrial(Program program, boolean commit, String referenceSource, UUID id) { + public BrAPITrial constructBrAPITrial(Program program, boolean commit, String referenceSource, UUID id, String expSeqValue) { BrAPIProgram brapiProgram = program.getBrapiProgram(); BrAPITrial trial = new BrAPITrial(); if( commit ){ @@ -128,6 +128,7 @@ public BrAPITrial constructBrAPITrial(Program program, boolean commit, String re trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.DEFAULT_OBSERVATION_LEVEL, getExpUnit()); trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, getExpType()); + trial.putAdditionalInfoItem( BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER, expSeqValue); return trial; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 48dac3659..1905e2e81 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; @@ -32,7 +33,6 @@ import org.brapi.v2.model.pheno.*; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -63,6 +63,7 @@ public class ExperimentProcessor implements Processor { @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; + private DSLContext dsl; private BrAPITrialDAO brapiTrialDAO; private BrAPILocationDAO brAPILocationDAO; @@ -198,9 +199,10 @@ private void getNewBrapiData(List importRows, Program program, bool for (BrAPIImport row : importRows) { ExperimentObservation importRow = (ExperimentObservation) row; - PendingImportObject trialPIO = createTrialPIO(program, commit, importRow); + PendingImportObject trialPIO = createTrialPIO(program, commit, importRow, expSeqValue); this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); + //TODO move this above the creation of trialPIO (see BI-1189 tech spec 4.b)? PendingImportObject locationPIO = createLocationPIO(importRow); this.locationByName.put(importRow.getEnvLocation(), locationPIO); @@ -420,14 +422,14 @@ private PendingImportObject createLocationPIO(ExperimentObservati return pio; } - private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow) { + private PendingImportObject createTrialPIO(Program program, boolean commit, ExperimentObservation importRow, String expSeqValue) { PendingImportObject pio = null; if( trialByNameNoScope.containsKey( importRow.getExpTitle()) ) { pio = trialByNameNoScope.get( importRow.getExpTitle() ) ; } else { UUID id = UUID.randomUUID(); - BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit, BRAPI_REFERENCE_SOURCE, id); + BrAPITrial newTrial = importRow.constructBrAPITrial(program, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); } return pio; @@ -583,29 +585,30 @@ private Map> initialize_existingGerm private Map> initialize_studyByNameNoScope(Program program, List experimentImportRows) { Map> studyByNameNoScope = new HashMap<>(); - List uniqueStudyNames = experimentImportRows.stream() - .map(experimentObservation -> { - String envName = experimentObservation.getEnv(); - PendingImportObject trial = trialByNameNoScope.get(experimentObservation.getExpTitle()); - if(trial.getBrAPIObject().getAdditionalInfo() != null) { - envName = Utilities.appendProgramKey(envName, program.getName(), trial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); - } - - return envName; - }) - .distinct() - .filter(Objects::nonNull) - .collect(Collectors.toList()); + + + if( this.trialByNameNoScope.size()!=1){ + return studyByNameNoScope; + } + ExperimentObservation experimentObservation = experimentImportRows.get(0); + + PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); + List experimentRefs = trial.getBrAPIObject().getExternalReferences(); + Optional experimentIDRef = experimentRefs.stream() + .filter(this::isRefSource) + .findFirst(); + if( experimentIDRef.isEmpty()){ + throw new InternalServerException("An Experiment ID was not found any of the external references"); + } + + //get experimentID + String experimentIDStr = experimentIDRef.get().getReferenceID(); List existingStudies; try { - existingStudies = brAPIStudyDAO.getStudyByName(uniqueStudyNames, program); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(experimentIDStr), program); existingStudies.forEach(existingStudy -> { - // I don't think this is needed anymore -// String studySequence = null; -// if ((existingStudy.getAdditionalInfo()!=null) && (existingStudy.getAdditionalInfo().get("studySequence") != null )) { -// studySequence = existingStudy.getAdditionalInfo().get("studySequence").getAsString(); -// } + existingStudy.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey())); studyByNameNoScope.put( existingStudy.getStudyName(), @@ -716,4 +719,8 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { String seasonDbId = (targetSeason==null) ? null : targetSeason.getSeasonDbId(); return seasonDbId; } + + private boolean isRefSource(BrAPIExternalReference brAPIExternalReference) { + return brAPIExternalReference.getReferenceSource().equals(BRAPI_REFERENCE_SOURCE); + } } \ No newline at end of file From 5d3ac0066a5540324ed1cc3427958a29e3e64110 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 25 Jul 2022 11:23:47 -0400 Subject: [PATCH 23/24] [BI-1189] fixed bug. Added ExternalReferenceSource.java to emumerate reference sourse names --- .../ExperimentObservation.java | 13 +++++++------ .../services/ExternalReferenceSource.java | 18 ++++++++++++++++++ .../processors/ExperimentProcessor.java | 3 ++- 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 2e05620b2..d396a17b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -26,6 +26,7 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -261,10 +262,10 @@ private List getBrAPIExternalReferences( Program program, String referenceSourceBaseName, UUID trialId, UUID studyId, UUID obsUnitId) { List refs = new ArrayList<>(); - addReference(refs, program.getId(), referenceSourceBaseName, "programs"); - if( trialId != null ) { addReference(refs, trialId, referenceSourceBaseName, "trials"); } - if( studyId != null ) { addReference(refs, studyId, referenceSourceBaseName, "studies"); } - if( obsUnitId != null ) { addReference(refs, obsUnitId, referenceSourceBaseName, "observationunits"); } + addReference(refs, program.getId(), referenceSourceBaseName, ExternalReferenceSource.PROGRAMS); + if( trialId != null ) { addReference(refs, trialId, referenceSourceBaseName, ExternalReferenceSource.TRIALS); } + if( studyId != null ) { addReference(refs, studyId, referenceSourceBaseName, ExternalReferenceSource.STUDIES); } + if( obsUnitId != null ) { addReference(refs, obsUnitId, referenceSourceBaseName, ExternalReferenceSource.OBSERVATION_UNITS); } return refs; } @@ -283,10 +284,10 @@ private List getObsUnitExternalReferences( } - private void addReference(List refs, UUID uuid, String referenceBaseNameSource, String refSourceName) { + private void addReference(List refs, UUID uuid, String referenceBaseNameSource, ExternalReferenceSource refSourceName) { BrAPIExternalReference reference; reference = new BrAPIExternalReference(); - reference.setReferenceSource( String.format("%s/%s", referenceBaseNameSource, refSourceName) ); + reference.setReferenceSource( String.format("%s/%s", referenceBaseNameSource, refSourceName.getName()) ); reference.setReferenceID(uuid.toString()); refs.add(reference); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java new file mode 100644 index 000000000..376cc1290 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services; + +import lombok.Getter; + +@Getter +public enum ExternalReferenceSource { + PROGRAMS("programs"), + TRIALS("trials"), + STUDIES("studies"), + OBSERVATION_UNITS("observationunits"); + + private String name; + + ExternalReferenceSource(String name) { + this.name = name; + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 1905e2e81..c50b68d53 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -42,6 +42,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; @@ -721,6 +722,6 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { } private boolean isRefSource(BrAPIExternalReference brAPIExternalReference) { - return brAPIExternalReference.getReferenceSource().equals(BRAPI_REFERENCE_SOURCE); + return brAPIExternalReference.getReferenceSource().equals( String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()) ); } } \ No newline at end of file From 6bfe6499482478badbc561ebb549616053e772d9 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 29 Jul 2022 11:20:06 -0400 Subject: [PATCH 24/24] [BI-1189] two small changes suggested by Tim in PR --- .../importer/services/processors/ExperimentProcessor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c50b68d53..3e82e87b1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -203,7 +203,6 @@ private void getNewBrapiData(List importRows, Program program, bool PendingImportObject trialPIO = createTrialPIO(program, commit, importRow, expSeqValue); this.trialByNameNoScope.put(importRow.getExpTitle(), trialPIO); - //TODO move this above the creation of trialPIO (see BI-1189 tech spec 4.b)? PendingImportObject locationPIO = createLocationPIO(importRow); this.locationByName.put(importRow.getEnvLocation(), locationPIO); @@ -596,7 +595,7 @@ private Map> initialize_studyByNameNoSco PendingImportObject trial = this.trialByNameNoScope.get(experimentObservation.getExpTitle()); List experimentRefs = trial.getBrAPIObject().getExternalReferences(); Optional experimentIDRef = experimentRefs.stream() - .filter(this::isRefSource) + .filter(this::isTrialRefSource) .findFirst(); if( experimentIDRef.isEmpty()){ throw new InternalServerException("An Experiment ID was not found any of the external references"); @@ -721,7 +720,7 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { return seasonDbId; } - private boolean isRefSource(BrAPIExternalReference brAPIExternalReference) { + private boolean isTrialRefSource(BrAPIExternalReference brAPIExternalReference) { return brAPIExternalReference.getReferenceSource().equals( String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()) ); } } \ No newline at end of file