diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 821c39a98..27723e9d5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -47,6 +47,7 @@ public final class BrAPIAdditionalInfoFields { public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; public static final String TREATMENTS = "treatments"; public static final String GID = "gid"; + public static final String CHANGELOG = "changeLog"; public static final String ENV_YEAR = "envYear"; public static final String GERMPLASM_UUID = "germplasmId"; public static final String SAMPLE_ORGANISM = "organism"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 7083cae0e..9c9ed184b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.daos; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; @@ -24,11 +25,13 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationSingleResponse; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; import javax.inject.Inject; @@ -38,6 +41,7 @@ import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @Singleton +@Slf4j public class BrAPIObservationDAO { private ProgramDAO programDAO; @@ -113,4 +117,25 @@ public List createBrAPIObservations(List brA return brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); } + public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation observation, UUID programId) throws ApiException { + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); + ApiResponse response; + BrAPIObservation updatedObservation = null; + try { + response = api.observationsObservationDbIdPut(dbId, observation); + if (response != null) { + BrAPIObservationSingleResponse body = response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); + } + updatedObservation = body.getResult(); + if (updatedObservation == null) { + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); + } + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e)); + } + return updatedObservation; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java b/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java index bb176bd0d..d54b509ef 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java @@ -21,6 +21,7 @@ @Getter public enum ImportFieldTypeEnum { + BOOLEAN("BOOLEAN"), TEXT("TEXT"), NUMERICAL("NUMERICAL"), INTEGER("INTEGER"), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 2bf412955..1d520371c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -49,5 +49,5 @@ default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException; + throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java new file mode 100644 index 000000000..e2ec109ed --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java @@ -0,0 +1,39 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.Getter; +import lombok.Setter; + +import java.util.UUID; + +@Getter +@Setter +public class ChangeLogEntry { + private String originalValue; + private String reasonForChange; + private UUID changedBy; + private String dateOfChange; + + public ChangeLogEntry(String originalValue, String reasonForChange, UUID changedBy, String dateOfChange) { + this.originalValue = originalValue; + this.reasonForChange = reasonForChange; + this.changedBy = changedBy; + this.dateOfChange = dateOfChange; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index aa425a2da..cd795564a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,18 +18,15 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; -import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImport; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.services.processors.*; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import javax.inject.Inject; @@ -70,7 +67,7 @@ public String getMissingColumnMsg(String columnName) { @Override public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, ValidatorException, ApiException, MissingRequiredInfoException { + throws Exception { ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index f45cd83a3..290c046ad 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -47,6 +47,13 @@ @ImportConfigMetadata(id = "ExperimentImport", name = "Experiment Import", description = "This import is used to create Observation Unit and Experiment data") public class ExperimentObservation implements BrAPIImport { + @ImportFieldType(type = ImportFieldTypeEnum.BOOLEAN, collectTime = ImportCollectTimeEnum.UPLOAD) + @ImportFieldMetadata(id = "overwrite", name = "Overwrite", description = "Boolean flag to overwrite existing observation") + private String overwrite; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) + @ImportFieldMetadata(id="overwriteReason", name="Overwrite Reason", description="Description of the reason for overwriting existing observations") + private String overwriteReason; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) @ImportFieldMetadata(id = "germplasmName", name = Columns.GERMPLASM_NAME, description = "Name of germplasm") diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index e4084eae4..b4eac6b96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -18,24 +18,21 @@ package org.breedinginsight.brapps.importer.model.imports.germplasm; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.services.processors.*; +import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; +import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.*; +import java.util.List; @Singleton @Slf4j @@ -66,7 +63,7 @@ public String getImportTypeId() { @Override public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException { + throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 637fa7ce3..434626e68 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapps.importer.model.imports.sample; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; @@ -28,10 +27,6 @@ import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import javax.inject.Inject; @@ -69,7 +64,7 @@ public ImportPreviewResponse process(List brAPIImports, Program program, ImportUpload upload, User user, - Boolean commit) throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException { + Boolean commit) throws Exception { List processors = List.of(sampleProcessorProvider.get()); return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index 5a3a2289f..a3535d13c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -53,6 +53,12 @@ public class MappingManager { private ImportConfigManager configManager; + public static String wrongDataTypeMsg = "Column name \"%s\" must be integer type, but non-integer type provided."; + public static String blankRequiredField = "Required field \"%s\" cannot contain empty values"; + public static String missingColumn = "Column name \"%s\" does not exist in file"; + public static String missingUserInput = "User input, \"%s\" is required"; + public static String wrongUserInputDataType = "User input, \"%s\" must be an %s"; + @Inject MappingManager(ImportConfigManager configManager) { this.configManager = configManager; @@ -339,13 +345,13 @@ private void mapUserInputField(Object parent, Field field, Map u // Only supports user input at the top level of an object at the moment. No nested objects. Map String fieldId = metadata.id(); - if (!userInput.containsKey(fieldId) && required != null) { + if ((userInput == null || !userInput.containsKey(fieldId)) && required != null) { throw new UnprocessableEntityException(importService.getMissingUserInputMsg(metadata.name())); } else if (required != null && userInput.containsKey(fieldId) && userInput.get(fieldId).toString().isBlank()) { throw new UnprocessableEntityException(importService.getMissingUserInputMsg(metadata.name())); } - else if (userInput.containsKey(fieldId)) { + else if (userInput != null && userInput.containsKey(fieldId)) { String value = userInput.get(fieldId).toString(); if (!isCorrectType(type.type(), value)) { throw new UnprocessableEntityException(importService.getWrongUserInputDataTypeMsg(metadata.name(), type.type().toString().toLowerCase())); @@ -371,11 +377,14 @@ private Boolean isCorrectType(ImportFieldTypeEnum expectedType, String value) { if (!value.isBlank()) { if (expectedType == ImportFieldTypeEnum.INTEGER) { try { - Integer d = Integer.parseInt(value); + Integer.parseInt(value); } catch (NumberFormatException nfe) { return false; } } + if (expectedType == ImportFieldTypeEnum.BOOLEAN && !String.valueOf(Boolean.parseBoolean(value)).equals(value)) { + return false; + } } return true; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e5ec4a5c5..d35da62d8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -16,7 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors; -import com.google.gson.JsonElement; +import com.google.gson.Gson; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; @@ -27,6 +28,8 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; @@ -45,6 +48,7 @@ import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; @@ -58,6 +62,7 @@ import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; @@ -67,13 +72,13 @@ import tech.tablesaw.columns.Column; import javax.inject.Inject; +import javax.validation.Valid; import java.math.BigDecimal; import java.math.BigInteger; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -82,6 +87,10 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; + private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s.

" + + "If you’re trying to add these units to the experiment, please create a new environment" + + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + + "for each experiment unit)."; private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + @@ -91,7 +100,9 @@ public class ExperimentProcessor implements Processor { private static final String TIMESTAMP_PREFIX = "TS:"; private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; private static final String COMMA_DELIMITER = ","; - private static final String BLANK_FIELD = "Required field is blank"; + private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; + private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; + private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment"; private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment"; @@ -127,12 +138,13 @@ public class ExperimentProcessor implements Processor { private Map> observationUnitByNameNoScope = null; private final Map> observationByHash = new HashMap<>(); - + private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; // Associates timestamp columns to associated phenotype column name for ease of storage private final Map> timeStampColByPheno = new HashMap<>(); + private final Gson gson; @Inject public ExperimentProcessor(DSLContext dsl, @@ -156,6 +168,7 @@ public ExperimentProcessor(DSLContext dsl, this.brAPIListDAO = brAPIListDAO; this.ontologyService = ontologyService; this.fileMappingUtil = fileMappingUtil; + this.gson = new JSON().getGson(); } @Override @@ -202,7 +215,7 @@ public Map process( Table data, Program program, User user, - boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { + boolean commit) throws UnprocessableEntityException, ApiException, ValidatorException { log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -220,7 +233,7 @@ public Map process( } } - List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols, validationErrors); + List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval for (Column tsColumn : timestampCols) { @@ -232,7 +245,7 @@ public Map process( prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); - validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit); + validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit, user); if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); @@ -256,6 +269,9 @@ public void postBrapiData(Map mappedBrAPIImport, Program Map mutatedTrialsById = ProcessorData .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + List newLocations = ProcessorData.getNewObjects(this.locationByName) .stream() .map(location -> ProgramLocationRequest.builder() @@ -277,6 +293,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + // filter out observations with no 'value' so they will not be saved List newObservations = ProcessorData.getNewObjects(this.observationByHash) .stream() @@ -299,7 +316,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program } List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); - // set the DbId to the for each newly created trial + // set the DbId to the for each newly created location for (ProgramLocation createdLocation : createdLocations) { String createdLocationName = createdLocation.getName(); this.locationByName.get(createdLocationName) @@ -373,6 +390,28 @@ public void postBrapiData(Map mappedBrAPIImport, Program throw new InternalServerException(e.getMessage(), e); } }); + + mutatedObservationByDbId.forEach((id, observation) -> { + try { + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + if (!observation.getValue().equals(updatedObs.getValue()) || !observation.getObservationTimeStamp().isEqual(updatedObs.getObservationTimeStamp())) { + String message; + if(!observation.getValue().equals(updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + log.debug("experiment import complete"); } @@ -401,7 +440,7 @@ private void prepareDataForValidation(List importRows, List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { + private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols) { Set varNames = phenotypeCols.stream() .map(Column::name) .collect(Collectors.toSet()); @@ -459,7 +498,7 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws UnprocessableEntityException, ApiException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -523,7 +562,7 @@ private void initNewBrapiData(List importRows, List> phen } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - fetchOrCreateObservationPIO(program, user, importRow, column.name(), column.getString(rowNum), dateTimeValue, commit, seasonDbId, obsUnitPIO); + fetchOrCreateObservationPIO(program, user, importRow, column, rowNum, dateTimeValue, commit, seasonDbId, obsUnitPIO, referencedTraits); } } } @@ -547,10 +586,9 @@ private String getObservationHash(String observationUnitName, String variableNam return DigestUtils.sha256Hex(concat); } - private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, - List> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException { + private void validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols, boolean commit, User user) { //fetching any existing observations for any OUs in the import - Map existingObsByObsHash = fetchExistingObservations(referencedTraits, program); CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); for ( Trait trait: referencedTraits) { colVarMap.put(trait.getObservationVariableName(),trait); @@ -563,18 +601,17 @@ private ValidationErrors validateFields(List importRows, Validation validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); } validateTestOrCheck(importRow, validationErrors, rowNum); - + //TODO: providing obs unit ID does not supersede import row inout data as expected and needs to be fixed //Check if existing environment. If so, ObsUnitId must be assigned - if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) - && (StringUtils.isBlank(importRow.getObsUnitID()))) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } +// if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) +// && (StringUtils.isBlank(importRow.getObsUnitID()))) { +// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); +// } validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } - return validationErrors; } private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { @@ -598,10 +635,16 @@ private Map fetchExistingObservations(List refe .map(PendingImportObject::getBrAPIObject) .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - observationUnitByNameNoScope.values().forEach(ou -> { - if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { - ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); - ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); } }); @@ -625,17 +668,71 @@ private Map fetchExistingObservations(List refe .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, CaseInsensitiveMap colVarMap, Map existingObservations) { + private void validateObservations(ValidationErrors validationErrors, + int rowNum, + ExperimentObservation importRow, + List> phenotypeCols, + CaseInsensitiveMap colVarMap, + boolean commit, + User user) { phenotypeCols.forEach(phenoCol -> { - var importHash = getImportObservationHash(importRow, phenoCol.name()); - if(existingObservations.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(rowNum)) && !existingObservations.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + String importHash = getImportObservationHash(importRow, phenoCol.name()); + String importObsValue = phenoCol.getString(rowNum); + + // error if import observation data already exists and user has not selected to overwrite + if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { addRowError( phenoCol.name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), validationErrors, rowNum ); - } else if(existingObservations.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || existingObservations.get(importHash).getValue().equals(phenoCol.getString(rowNum)))) { - BrAPIObservation existingObs = existingObservations.get(importHash); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is either empty or the + // same as has been committed prior to import + } else if(existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || + isObservationMatched(importHash, importObsValue, phenoCol, rowNum))) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); observationByHash.get(importHash).setBrAPIObject(existingObs); @@ -680,7 +777,12 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in .getState(); ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); - String errorMessage = BLANK_FIELD; + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { validateRequiredCell(importRow.getGid(), Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); @@ -711,7 +813,8 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); } } else { - validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); + //TODO: include this step once user-supplied obs unit id correctly supersedes other row data + //validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); } } @@ -757,6 +860,25 @@ private Map generateStatisticsMap(List preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -769,12 +891,20 @@ private Map generateStatisticsMap(List getGidPOI(ExperimentObservation impo return null; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) { + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws UnprocessableEntityException, ApiException { PendingImportObject pio; String key = createObservationUnitKey(importRow); if (this.observationUnitByNameNoScope.containsKey(key)) { @@ -827,33 +957,94 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new UnprocessableEntityException(String.format(EXISTING_ENV, importRow.getExpUnitId(), + Utilities.removeProgramKeyAndUnknownAdditionalData(studyPIO.getBrAPIObject().getStudyName(), program.getKey()))); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } this.observationUnitByNameNoScope.put(key, pio); } return pio; } + boolean isTimestampMatched(String observationHash, String timeStamp) { + OffsetDateTime priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + if (priorStamp == null) { + return timeStamp == null; + } + return priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + } + + boolean isValueMatched(String observationHash, String value) { + if (existingObsByObsHash.get(observationHash).getValue() == null) { + return value == null; + } + return existingObsByObsHash.get(observationHash).getValue().equals(value); + } + + boolean isObservationMatched(String observationHash, String value, Column phenoCol, Integer rowNum) { + if (timeStampColByPheno.isEmpty() || !timeStampColByPheno.containsKey(phenoCol.name())) { + return isValueMatched(observationHash, value); + } else { + String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); + return isTimestampMatched(observationHash, importObsTimestamp) && isValueMatched(observationHash, value); + } + } - private PendingImportObject fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - String variableName, - String value, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO) { + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + Column column, + Integer rowNum, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + List referencedTraits) throws ApiException { PendingImportObject pio; + BrAPIObservation newObservation; + String variableName = column.name(); + String value = column.getString(rowNum); String key = getImportObservationHash(importRow, variableName); - if (this.observationByHash.containsKey(key)) { - pio = observationByHash.get(key); - } else { + existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + if (existingObsByObsHash.containsKey(key)) { + if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, column, rowNum)){ + + // prior observation with updated value + newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); + if (!isValueMatched(key, value)){ + newObservation.setValue(value); + } else if (!isTimestampMatched(key, timeStampValue)) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } + pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + } else { + + // prior observation + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); + } + + observationByHash.put(key, pio); + } else if (!this.observationByHash.containsKey(key)){ + + // new observation PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); + newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { @@ -865,13 +1056,17 @@ private PendingImportObject fetchOrCreateObservationPIO(Progra pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); } - return pio; } private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { BrAPIListDetails details = pio.getBrAPIObject(); referencedTraits.forEach(trait -> { - String id = trait.getRawObservationVariableName(); + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + // Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + if (trait.getFullName() == null) { + id = trait.getObservationVariableName(); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { details.getData().add(id); } @@ -881,7 +1076,7 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } }); } - private PendingImportObject fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { PendingImportObject pio; PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); String name = String.format("Observation Dataset [%s-%s]", @@ -908,7 +1103,6 @@ private PendingImportObject fetchOrCreateDatasetPIO(Experiment obsVarDatasetByName.put(name, pio); } addObsVarsToDatasetDetails(pio, referencedTraits, program); - return pio; } private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { @@ -927,7 +1121,7 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b // It is assumed that the study has only one season, And that the Years and not // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season if (commit) { if(StringUtils.isNotBlank(year)) { String seasonID = this.yearToSeasonDbId(year, program.getId()); @@ -937,7 +1131,6 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b addYearToStudyAdditionalInfo(program, newStudy, year); } - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); this.studyByNameNoScope.put(importRow.getEnv(), pio); } @@ -975,16 +1168,13 @@ private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Str } } - private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { PendingImportObject pio; - if (locationByName.containsKey((importRow.getEnvLocation()))) { - pio = locationByName.get(importRow.getEnvLocation()); - } else { + if (! locationByName.containsKey((importRow.getEnvLocation()))) { ProgramLocation newLocation = importRow.constructProgramLocation(); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); this.locationByName.put(importRow.getEnvLocation(), pio); } - return pio; } private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) throws UnprocessableEntityException { @@ -1268,9 +1458,7 @@ private void initializeStudiesForExistingObservationUnits(Program program, Map studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> { - processAndCacheStudy(study, program, studyByName); - }); + studies.forEach(study -> processAndCacheStudy(study, program, studyByName)); } private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { @@ -1340,7 +1528,7 @@ private Map> initializeObsVarDatas BrAPIListDetails dataSetDetails = brAPIListDAO .getListById(existingDatasets.get(0).getListDbId(), program.getId()) .getResult(); - processAndCacheObsVarDataset(dataSetDetails, program, obsVarDatasetByName); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1364,7 +1552,7 @@ private Optional> getTrialPIO(List> obsVarDatasetByName) { + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); @@ -1422,9 +1610,15 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = this.seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } studyByName.put( Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.getReferenceID()))); + new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), UUID.fromString(xref.getReferenceID()))); } private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index e4c2e067a..820626676 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -23,7 +23,6 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -55,7 +54,7 @@ public interface Processor { Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) - throws ValidatorException, MissingRequiredInfoException, ApiException; + throws Exception; /** * Given mapped brapi import with updates from prior dependencies, check if have everything needed diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java index 1f61dabe8..1961b33d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java @@ -18,7 +18,6 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -27,7 +26,6 @@ import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -53,7 +51,7 @@ public ProcessorManager(ImportStatusService statusService) { this.statusService = statusService; } - public ImportPreviewResponse process(List importRows, List processors, Table data, Program program, ImportUpload upload, User user, boolean commit) throws ValidatorException, ApiException, MissingRequiredInfoException { + public ImportPreviewResponse process(List importRows, List processors, Table data, Program program, ImportUpload upload, User user, boolean commit) throws Exception { this.processors = processors; diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 7aa390635..c885eaa97 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -24,6 +24,7 @@ import org.breedinginsight.model.Program; import org.flywaydb.core.api.migration.Context; +import java.lang.reflect.Field; import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; @@ -115,6 +116,42 @@ public static String removeProgramKey(String original, String programKey) { return removeProgramKey(original, programKey, null); } + /** + * Remove program key from fields visible on the front end. Mutates the original object and returns it. + * + * @param brapiInstance Object, an instance of a BrAPI Object + * @param brapiClass Class, the BrAPI class + * @param program + * @return Object, BrAPI instance formatted for display + */ + public static Object formatBrapiObjForDisplay(Object brapiInstance, Class brapiClass, Program program) throws RuntimeException { + List displayFields = new ArrayList<>(List.of( + "trialName", + "studyName", + "germplasmName", + "locationName", + "observationUnitName", + "observationVariableName")); + List fields = List.of(brapiClass.getDeclaredFields()); + for (Field field : fields) { + if (displayFields.contains(field.getName())) { + try { + field.setAccessible(true); + + // remove either of possible key formats, [%s-%s] and [%s] + String valueSansKeyAndInfo = removeProgramKeyAndUnknownAdditionalData((String) field.get(brapiInstance), program.getKey()); + String valueSansKey = removeProgramKey(valueSansKeyAndInfo, program.getKey()); + + // set the value without key or additional info + field.set(brapiInstance, valueSansKey); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + return brapiInstance; + } + public static String removeProgramKeyAndUnknownAdditionalData(String original, String programKey) { String keyValueRegEx = String.format(" \\[%s\\-.*\\]", programKey); String stripped = original.replaceAll(keyValueRegEx, ""); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 748299daf..e6b64ca49 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -86,6 +86,7 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { + private static final String OVERWRITE = "overwrite"; private FannyPack securityFp; private String mappingId; @@ -603,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Units are missing Observation Unit Id.")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create new observation unit")); } @Test