diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 3c3c6a093..039c0b58c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -449,7 +449,7 @@ private void processFile(String workflowId, List finalBrAPIImportLi progress.setMessage(e.getMessage()); progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); - }catch (ValidatorException e) { + } catch (ValidatorException e) { log.info("Validation errors: \n" + e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index ae015aab6..4c79478e8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -44,6 +44,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.model.Program; import org.breedinginsight.model.Scale; @@ -70,8 +71,7 @@ public class ExperimentUtilities { public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; public static final String UNMATCHED_COLUMN = "Ontology term(s) not found: "; - - + public static final String INVALID_OBS_UNIT_ID_ERROR = "Invalid ObsUnitID"; Gson gson; @@ -294,44 +294,104 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud } /** - * This method is responsible for collating unique ObsUnit IDs from the provided context data. + * Collates unique Observation Unit IDs from the import context. + * + * This method iterates through all import rows in the given context and + * extracts unique Observation Unit IDs (ObsUnit IDs) that are not null or blank. + * + * @param context The AppendOverwriteMiddlewareContext containing the import data. + * @return A Set of String containing all unique, non-null, non-blank Observation Unit IDs. * - * @param context the AppendOverwriteMiddlewareContext containing the import rows to process - * @return a Set of unique ObsUnit IDs collated from the import rows - * @throws IllegalStateException if any ObsUnit ID is repeated in the import rows - * @throws HttpStatusException if there is a mix of ObsUnit IDs for some but not all rows + * @implNote The method performs the following steps: + * 1. Initializes an empty HashSet to store unique ObsUnit IDs. + * 2. Iterates through each import row in the context. + * 3. For each row, checks if the ObsUnit ID is not null and not blank. + * 4. If valid, adds the ObsUnit ID to the set. + * 5. Returns the set of unique ObsUnit IDs. */ - public static Set collateReferenceOUIds(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException { + public static Set collateUniqueOUIds(AppendOverwriteMiddlewareContext context) { // Initialize variables to track the presence of ObsUnit IDs Set referenceOUIds = new HashSet<>(); - boolean hasNoReferenceUnitIds = true; - boolean hasAllReferenceUnitIds = true; + + // Iterate through the import rows to process ObsUnit IDs + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + if (importRow.getObsUnitID() != null && !importRow.getObsUnitID().isBlank()) { + referenceOUIds.add(importRow.getObsUnitID()); + } + } + return referenceOUIds; + } + + /** + * Validates Observation Unit ID values in the import context. + * + * This method checks each import row for the validity of its Observation Unit ID (ObsUnitID). + * It performs the following validations: + * 1. Checks if the ObsUnitID is null or blank. + * 2. Checks if the ObsUnitID is a duplicate within the import data. + * + * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. + * @throws HttpStatusException If there's an HTTP-related error during the validation process. + * @throws IllegalStateException If the system encounters an unexpected state during validation. + * + * @implNote The method performs the following steps: + * 1. Retrieves the ValidationErrors object from the context. + * 2. Initializes a HashSet to track unique ObsUnitIDs. + * 3. Iterates through each import row in the context. + * 4. For each row: + * - If ObsUnitID is null or blank, adds a "missing ObsUnitID" error. + * - If ObsUnitID is already in the set (duplicate), adds a "duplicate ObsUnitID" error. + * - Otherwise, adds the ObsUnitID to the set of unique IDs. + * 5. Errors are added using the addRowError method, specifying the OBS_UNIT_ID column and appropriate error messages. + */ + public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException { + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + Set referenceOUIds = new HashSet<>(); // Iterate through the import rows to process ObsUnit IDs for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - // Check if ObsUnitID is blank if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - // Set flag to indicate missing ObsUnit ID for current row - hasAllReferenceUnitIds = false; + // Check if ObsUnitID is blank + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID.getValue(), validationErrors, rowNum); } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - // Throw exception if ObsUnitID is repeated - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + // Check if ObsUnitID is repeated + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.DUPLICATE_OBS_UNIT_ID.getValue(), validationErrors, rowNum); } else { // Add ObsUnitID to referenceOUIds referenceOUIds.add(importRow.getObsUnitID()); - // Set flag to indicate presence of ObsUnit ID - hasNoReferenceUnitIds = false; } } + } - if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { - // Throw exception if there is a mix of ObsUnit IDs for some but not all rows - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR.getValue()); - } + /** + * Adds validation errors for observation units that were not found in the database. + * + * This method processes an EntityNotFoundException and adds corresponding validation errors + * to the context for each import row where the Observation Unit ID was not found. + * + * @param e The EntityNotFoundException containing information about missing Observation Unit IDs. + * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage. + * + * @implNote The method performs the following steps: + * 1. Retrieves the ValidationErrors object from the context. + * 2. Iterates through each import row in the context. + * 3. For each row, checks if its Observation Unit ID is in the set of missing entity IDs from the exception. + * 4. If a match is found, adds a validation error for that row, indicating an invalid Observation Unit ID. + * 5. The error is added using the addRowError method, specifying the OBS_UNIT_ID column and using a predefined error message. + */ + public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext context) { + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + List errors = new ArrayList<>(); - return referenceOUIds; + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + String rowObsUnitId = ((ExperimentObservation)context.getImportContext().getImportRows().get(rowNum)).getObsUnitID(); + if (e.getMissingEntityIds().contains(rowObsUnitId)) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum); + } + } } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java index 830de5828..2702f9c7a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java @@ -20,9 +20,11 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; import java.util.Optional; @@ -42,7 +44,7 @@ public interface BrAPIAction { * @return An Optional containing the relevant BrAPI state after executing the action. * @throws ApiException if an error occurs during the execution of the action. */ - Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException; + Optional> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException, EntityNotFoundException; /** * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java index 717a78f97..198c9c1ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java @@ -23,6 +23,8 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; +import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; import java.util.List; @@ -44,7 +46,7 @@ protected WorkflowReadInitialization(ExperimentImportEntity entity) { * @return an Optional containing the BrAPIState representing the completed read workflow * @throws ApiException if an error occurs during execution */ - public Optional> execute() throws ApiException { + public Optional> execute() throws ApiException, EntityNotFoundException { try { List fetchedMembers = entity.brapiRead(); entity.initializeWorkflow(fetchedMembers); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java index 84f8bcf75..6775e6d03 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java @@ -112,6 +112,10 @@ public static EmptyData emptyData(String brapiReferenceSource, return new EmptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); } + public static UndefinedDataset undefinedDataset() { + return new UndefinedDataset(); + } + @Bean @Prototype public InitialData initialDataBean(String brapiReferenceSource, @@ -173,5 +177,11 @@ public EmptyData emptyDataBean(String brapiReferenceSource, Program program) { return emptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService); } + + @Bean + @Prototype + public UndefinedDataset undefinedDatasetBean() { + return undefinedDataset(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java new file mode 100644 index 000000000..d30d917ec --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java @@ -0,0 +1,43 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic; + +import java.util.List; +import java.util.Optional; + +public class UndefinedDataset extends VisitedObservationData { + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + return null; + } + + @Override + public void updateTally(AppendStatistic statistic) { + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java index c98d9cf83..e4b2a4aa2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java @@ -19,9 +19,11 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; import java.util.List; @@ -43,7 +45,7 @@ public interface ExperimentImportEntity { * @return List of fetched entities * @throws ApiException if there is an issue with the API call */ - public List brapiRead() throws ApiException; + public List brapiRead() throws ApiException, EntityNotFoundException; /** * Commit objects changed by the workflow to the BrAPI service. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index f7c29264c..f9ec45a2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -26,6 +26,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -94,12 +95,12 @@ public List brapiPost(List members) * @throws ApiException if there is an issue with the API call */ @Override - public List brapiRead() throws ApiException { + public List brapiRead() throws ApiException, EntityNotFoundException { // Collect deltabreed-generated obs unit ids listed in the import Set obsUnitIds = cache.getReferenceOUIds(); // For each id fetch the observation unit from the brapi data store - return observationUnitService.getObservationUnitsByDbId(new HashSet<>(obsUnitIds), importContext.getProgram()); + return observationUnitService.getObservationUnitsById(new HashSet<>(obsUnitIds), importContext.getProgram()); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java index 64f471b88..a2edbdf65 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/AppendOverwriteIDValidation.java @@ -18,25 +18,70 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.BrAPIReadFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.action.WorkflowReadInitialization; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.services.exceptions.ValidatorException; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.Set; @Slf4j @Prototype public class AppendOverwriteIDValidation extends AppendOverwriteMiddleware { + WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; + BrAPIReadFactory brAPIReadFactory; + + @Inject + public AppendOverwriteIDValidation(BrAPIReadFactory brAPIReadFactory) { + this.brAPIReadFactory = brAPIReadFactory; + } @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { + brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); + // Initialize the validation error collection + Optional.ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors()).orElseGet(() -> { + context.getAppendOverwriteWorkflowContext().setValidationErrors(new ValidationErrors()); + return new ValidationErrors(); + }); + ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors(); + ExperimentUtilities.validateReferenceOUIdValues(context); // Check for missing or duplicate OU ids + Set uniqueOUIds = ExperimentUtilities.collateUniqueOUIds(context); + context.getAppendOverwriteWorkflowContext().setReferenceOUIds(uniqueOUIds); try { - context.getAppendOverwriteWorkflowContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); - } catch (HttpStatusException | IllegalStateException e) { + brAPIObservationUnitReadWorkflowInitialization.execute(); // Fetch the obs units from the BrAPi service + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + return processNext(context); + } catch (EntityNotFoundException e) { + /** + * Return an error response with a list of rows where the unique OU id was not found in the BrAPI service in + * addition to rows where there are missing or duplicate OU ids + */ + ExperimentUtilities.addValidationErrorsForObsUnitsNotFound(e, context); + context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(new ValidatorException(validationErrors))); + return this.compensate(context); + } catch (ApiException | ValidatorException e) { + /** + * If OUs were fetched for all unique reference ids but some of the reference ids failed validation, + * return an error response and a list of rows with duplicate or missing ids + * + * Return an error response if there was a problem connecting to the BrAPI service + */ context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); } - return processNext(context); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java index e65d11d15..2aa889fc2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/initialize/WorkflowInitialization.java @@ -30,14 +30,15 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.exceptions.ValidatorException; import javax.inject.Inject; @Slf4j @Prototype public class WorkflowInitialization extends AppendOverwriteMiddleware { - WorkflowReadInitialization brAPIObservationUnitReadWorkflowInitialization; WorkflowReadInitialization brAPITrialReadWorkflowInitialization; WorkflowReadInitialization brAPIStudyReadWorkflowInitialization; WorkflowReadInitialization locationReadWorkflowInitialization; @@ -51,7 +52,6 @@ public WorkflowInitialization(BrAPIReadFactory brAPIReadFactory) { } @Override public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext context) { - brAPIObservationUnitReadWorkflowInitialization = brAPIReadFactory.observationUnitWorkflowReadInitializationBean(context); brAPITrialReadWorkflowInitialization = brAPIReadFactory.trialWorkflowReadInitializationBean(context); brAPIStudyReadWorkflowInitialization = brAPIReadFactory.studyWorkflowReadInitializationBean(context); locationReadWorkflowInitialization = brAPIReadFactory.locationWorkflowReadInitializationBean(context); @@ -60,7 +60,6 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext log.debug("reading required BrAPI data from BrAPI service"); try { - brAPIObservationUnitReadWorkflowInitialization.execute(); brAPITrialReadWorkflowInitialization.execute(); brAPIStudyReadWorkflowInitialization.execute(); locationReadWorkflowInitialization.execute(); @@ -69,6 +68,9 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } catch (ApiException e) { context.getAppendOverwriteWorkflowContext().setProcessError(new MiddlewareException(e)); return this.compensate(context); + } catch (EntityNotFoundException e) { + // TODO: handle edge cases of missing brapi entities as needed + return this.compensate(context); } return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 7ee29e51f..770576b67 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -225,6 +225,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { Integer rowNum = i; ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + VisitedObservationData processedData = null; // Construct the pending import for the row Optional.ofNullable(context.getImportContext().getMappedBrAPIImport()).orElseGet(() -> { @@ -233,18 +234,28 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext }); PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); String unitId = row.getObsUnitID(); + String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); mappedImportRow.setTrial(context.getAppendOverwriteWorkflowContext().getPendingTrialByOUId().get(unitId)); mappedImportRow.setLocation(context.getAppendOverwriteWorkflowContext().getPendingLocationByOUId().get(unitId)); mappedImportRow.setStudy(context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId)); mappedImportRow.setObservationUnit(context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId)); mappedImportRow.setGermplasm(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId)); + /** + * Handle the edge case where a user imports with the append/overwrite workflow for an experiment + * without a dataset defined (i.e. no observation variables headers) and the import does not + * actually have new data to append + */ + if (phenotypeCols.isEmpty()) { + processedData = processedDataFactory.undefinedDatasetBean(); + updatePreviewStatistics(processedData, context, studyName, unitId); + } + // Assemble the pending observation data for all phenotypes for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); // Generate hash for looking up prior observation data - String studyName = context.getAppendOverwriteWorkflowContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); String unitName = context.getAppendOverwriteWorkflowContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); @@ -266,8 +277,6 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } - VisitedObservationData processedData = null; - // Is there prior observation data for this unit + var? if (observationByObsHash.containsKey(observationHash)) { @@ -275,9 +284,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if ( - isChanged(cellData, observation, cell.timestamp) - ) { + if (isChanged(cellData, observation, cell.timestamp)) { // Is prior data protected? /** @@ -356,13 +363,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e -> validationErrors.addError(rowNum + 2, e))); // +2 to account for header row and excel file 1-based row index // Update import preview statistics and set in the context - processedData.updateTally(statistic); - statistic.addEnvironmentName(studyName); - // TODO: change null values to actual data - // TODO: change signature to take two args, studyName and unitName - statistic.addObservationUnitId(null); - statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); - context.getAppendOverwriteWorkflowContext().setStatistic(statistic); + updatePreviewStatistics(processedData, context, studyName, unitId); // Construct a pending observation Optional> pendingProcessedData = Optional.ofNullable(processedData.constructPendingObservation()); @@ -402,4 +403,34 @@ private boolean isChanged(String cellData, BrAPIObservation observation, String } return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); } + + /** + * Updates the preview statistics for processed observation data. + * + * This method updates various statistical metrics related to the processed + * observation data and stores them in the provided context. + * + * @param processedData The VisitedObservationData object containing the processed observation data. + * @param context The AppendOverwriteMiddlewareContext object where the updated statistics will be stored. + * @param studyName The name of the study associated with the observation data. + * @param unitId The identifier of the observation unit. + * + * @implNote This method performs the following operations: + * 1. Updates the tally in the processedData object. + * 2. Adds the study name to the statistics. + * 3. Adds the observation unit ID to the statistics. + * 4. Adds the germplasm ID (GID) to the statistics, retrieved from the pending germplasm data in the context. + * 5. Sets the updated statistics in the context. + */ + private void updatePreviewStatistics(VisitedObservationData processedData, + AppendOverwriteMiddlewareContext context, + String studyName, + String unitId) { + // Update import preview statistics and set in the context + processedData.updateTally(statistic); + statistic.addEnvironmentName(studyName); + statistic.addObservationUnitId(unitId); + statistic.addGid(context.getAppendOverwriteWorkflowContext().getPendingGermplasmByOUId().get(unitId).getBrAPIObject().getAccessionNumber()); + context.getAppendOverwriteWorkflowContext().setStatistic(statistic); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/EntityNotFoundException.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/EntityNotFoundException.java new file mode 100644 index 000000000..c6a915833 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/EntityNotFoundException.java @@ -0,0 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.Getter; + +import java.util.Set; +@Getter +public class EntityNotFoundException extends Throwable { + private Set missingEntityIds; + + public EntityNotFoundException(Set missingEntityIds) { this.missingEntityIds = missingEntityIds; } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index b4f7caf90..2c0ae2e53 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -1,12 +1,26 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.services.processors.experiment.model; import com.fasterxml.jackson.annotation.JsonValue; import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Value; import lombok.extern.slf4j.Slf4j; -import javax.annotation.PostConstruct; - @Slf4j public class ExpImportProcessConstants { @@ -18,8 +32,11 @@ public class ExpImportProcessConstants { public enum ErrMessage { MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), - MISSING_OBS_UNIT_ID_ERROR("Required field is blank"), - PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + MISSING_OBS_UNIT_ID("Invalid ObsUnitID"), + PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"), + UNMATCHED_COLUMN("Ontology term(s) not found: "), + OBS_UNIT_NOT_FOUND("Invalid ObsUnitID"), + DUPLICATE_OBS_UNIT_ID("ObsUnitId is repeated"); private String value; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 720c93d25..d66507558 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -22,11 +22,11 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -35,8 +35,6 @@ import java.util.*; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; - @Singleton public class ObservationUnitService { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; @@ -61,21 +59,26 @@ public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { * @throws ApiException if an error occurs during the retrieval of observation units * @throws IllegalStateException if the retrieved observation units do not match the provided observation unit IDs */ - public List getObservationUnitsByDbId(Set obsUnitIds, Program program) throws ApiException, IllegalStateException { + public List getObservationUnitsById(Set obsUnitIds, Program program) throws ApiException, IllegalStateException, EntityNotFoundException { List brapiUnits = null; // Retrieve reference Observation Units based on IDs brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(obsUnitIds, program); - // If no BrAPI units are found, throw an IllegalStateException with an error message + // If no BrAPI units are found, throw an EntityNotFoundException with an error message if (obsUnitIds.size() != brapiUnits.size()) { Set missingIds = new HashSet<>(obsUnitIds); // Calculate missing IDs based on retrieved BrAPI units - missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + //missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + missingIds.removeAll(brapiUnits.stream() + .map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(BrAPIExternalReference::getReferenceId).collect(Collectors.toSet())); // Throw exception with missing IDs information - throw new IllegalStateException(ExperimentUtilities.UNMATCHED_COLUMN + String.join(COMMA_DELIMITER, missingIds)); + throw new EntityNotFoundException(missingIds); } return brapiUnits;