From cdd3d688d3ece6313fcc012e3a242ea74f46f67d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:49:42 -0400 Subject: [PATCH 01/61] Initial work on refactor modeling skeleton --- .../ExperimentImportService.java | 2 + .../experiment/ExperimentProcessor.java | 42 +++++ .../experiment/ExperimentUtilities.java | 20 +++ .../AppendOverwritePhenotypesWorkflow.java | 25 +++ .../experiment/create/model/ExistingData.java | 18 +++ .../steps/GetExistingProcessingStep.java | 148 ++++++++++++++++++ .../experiment/create/steps/ProcessStep.java | 15 ++ .../workflow/CreateNewExperimentWorkflow.java | 44 ++++++ .../experiment/model/ImportContext.java | 28 ++++ .../experiment/model/ProcessedData.java | 11 ++ .../experiment/pipeline/Pipeline.java | 18 +++ .../experiment/pipeline/ProcessingStep.java | 5 + .../workflow/ExperimentWorkflowFactory.java | 71 +++++++++ .../experiment/workflow/Workflow.java | 10 ++ 14 files changed, 457 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..b7d9248d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -71,6 +71,8 @@ public ImportPreviewResponse process(List brAPIImports, Table data, ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); + Processor experimentProcessor = experimentProcessorProvider.get(); + //response = experimentProcessor.process(upload, brAPIImports, data, program, user, commit); response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); return response; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java new file mode 100644 index 000000000..8207086fe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentProcessor.java @@ -0,0 +1,42 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.ExperimentWorkflowFactory; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +@Slf4j +@Prototype +public class ExperimentProcessor { + + private final ExperimentWorkflowFactory experimentWorkflowFactory; + + @Inject + public ExperimentProcessor(ExperimentWorkflowFactory experimentWorkflowFactory) { + this.experimentWorkflowFactory = experimentWorkflowFactory; + } + + public Map process(ImportContext context) + throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { + + // determine which workflow to use based on the import context + Workflow workflow = experimentWorkflowFactory.getWorkflow(context); + log.info("Importing experiment data using workflow: " + workflow.getName()); + + ProcessedData output = workflow.process(context); + + + return new HashMap<>(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java new file mode 100644 index 000000000..bd90ee222 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; + +import java.util.List; +import java.util.stream.Collectors; + +public class ExperimentUtilities { + + public static final CharSequence COMMA_DELIMITER = ","; + + public static List importRowsToExperimentObservations(List importRows) { + return importRows.stream() + .map(trialImport -> (ExperimentObservation) trialImport) + .collect(Collectors.toList()); + } + + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..970fc25f3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,25 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; + +@Prototype +public class AppendOverwritePhenotypesWorkflow implements Workflow { + + @Override + public ProcessedData process(ImportContext context) { + + // validate that all rows have an ObsUnitID + + // TODO: implement + return new ProcessedData(); + } + + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java new file mode 100644 index 000000000..99113cc9f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ExistingData { + private Map> observationUnitByNameNoScope; + // TODO: add rest of fields +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java new file mode 100644 index 000000000..fc69c8105 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -0,0 +1,148 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Prototype +@Slf4j +public class GetExistingProcessingStep implements ProcessingStep { + + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + @Override + public ExistingData process(ImportContext input) { + + List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); + Program program = input.getProgram(); + + // getExisting + Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + // TODO: populate rest of data + + ExistingData existing = ExistingData.builder().observationUnitByNameNoScope(observationUnitByNameNoScope).build(); + + return existing; + } + + /** + * Initializes the observation units for the given program and experimentImportRows. + * + * @param program The program object + * @param experimentImportRows A list of ExperimentObservation objects + * @return A map of Observation Unit IDs to PendingImportObject objects + * + * @throws InternalServerException + * @throws IllegalStateException + */ + private Map> initializeObservationUnits(Program program, List experimentImportRows) { + Map> observationUnitByName = new HashMap<>(); + + Map rowByObsUnitId = new HashMap<>(); + experimentImportRows.forEach(row -> { + if (StringUtils.isNotBlank(row.getObsUnitID())) { + if(rowByObsUnitId.containsKey(row.getObsUnitID())) { + throw new IllegalStateException("ObsUnitId is repeated: " + row.getObsUnitID()); + } + rowByObsUnitId.put(row.getObsUnitID(), row); + } + }); + + try { + List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + if (existingObsUnits.size() == rowByObsUnitId.size()) { + existingObsUnits.forEach(brAPIObservationUnit -> { + processAndCacheObservationUnit(brAPIObservationUnit, refSource, program, observationUnitByName, rowByObsUnitId); + + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); + row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); + row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); + }); + } else { + List missingIds = new ArrayList<>(rowByObsUnitId.keySet()); + missingIds.removeAll(existingObsUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList())); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return observationUnitByName; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Adds a new map entry to observationUnitByName based on the brAPIObservationUnit passed in and sets the + * expUnitId in the rowsByObsUnitId map. + * + * @param brAPIObservationUnit the BrAPI observation unit object + * @param refSource the reference source + * @param program the program object + * @param observationUnitByName the map of observation units by name (will be modified in place) + * @param rowByObsUnitId the map of rows by observation unit ID (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservationUnit, String refSource, Program program, + Map> observationUnitByName, + Map rowByObsUnitId) { + BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); + row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); + observationUnitByName.put(createObservationUnitKey(row), + new PendingImportObject<>(ImportObjectState.EXISTING, + brAPIObservationUnit, + UUID.fromString(idRef.getReferenceId()))); + } + + private String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + private String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java new file mode 100644 index 000000000..fa483a7c4 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java @@ -0,0 +1,15 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.steps; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; + +public class ProcessStep implements ProcessingStep { + + @Override + public ProcessedData process(ExistingData input) { + + // TODO: implement + return new ProcessedData(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java new file mode 100644 index 000000000..06238aaab --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,44 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.context.annotation.Prototype; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.steps.GetExistingProcessingStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.steps.ProcessStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.Pipeline; +import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Prototype +@Slf4j +public class CreateNewExperimentWorkflow implements Workflow { + + private final Provider getExistingStepProvider; + private final Provider processStepProvider; + + @Inject + public CreateNewExperimentWorkflow(Provider getExistingStepProvider, + Provider processStepProvider) { + this.getExistingStepProvider = getExistingStepProvider; + this.processStepProvider = processStepProvider; + } + + @Override + public ProcessedData process(ImportContext context) { + + Pipeline pipeline = new Pipeline<>(getExistingStepProvider.get()) + .addProcessingStep(processStepProvider.get()); + ProcessedData processed = pipeline.execute(context); + + // TODO: return actual data + return processed; + } + + @Override + public String getName() { + return "CreateNewExperimentWorkflow"; + } +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java new file mode 100644 index 000000000..0738d6a1e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ImportContext.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java new file mode 100644 index 000000000..4c8fe464d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -0,0 +1,11 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; + +@Getter +@Setter +@Builder +@ToString +@NoArgsConstructor +public class ProcessedData { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java new file mode 100644 index 000000000..09b657a89 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/Pipeline.java @@ -0,0 +1,18 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; + +public class Pipeline { + + private final ProcessingStep currentStep; + + public Pipeline(ProcessingStep currentStep) { + this.currentStep = currentStep; + } + + public Pipeline addProcessingStep(ProcessingStep newStep) { + return new Pipeline<>(input -> newStep.process(currentStep.process(input))); + } + + public O execute(I input) { + return currentStep.process(input); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java new file mode 100644 index 000000000..2407e646e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/pipeline/ProcessingStep.java @@ -0,0 +1,5 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.pipeline; + +public interface ProcessingStep { + O process(I input); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java new file mode 100644 index 000000000..7f0b741fe --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/ExperimentWorkflowFactory.java @@ -0,0 +1,71 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.AppendOverwritePhenotypesWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.CreateNewExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +public class ExperimentWorkflowFactory { + + private final Provider createNewExperimentWorkflowProvider; + private final Provider appendOverwritePhenotypesWorkflowProvider; + + @Inject + public ExperimentWorkflowFactory(Provider createNewExperimentWorkflowProvider, + Provider appendOverwritePhenotypesWorkflowProvider) { + this.createNewExperimentWorkflowProvider = createNewExperimentWorkflowProvider; + this.appendOverwritePhenotypesWorkflowProvider = appendOverwritePhenotypesWorkflowProvider; + } + + /** + * Retrieves the appropriate workflow based on the provided import context. Validation will be done + * in selected workflow, not here. For example will not check if file has ObsUnitIDs that all rows have one. + * We are just checking the basic condition for what type of workflow to return. + * + * @param context import context containing import rows + * @return the workflow to be used for processing the import rows + */ + public Workflow getWorkflow(ImportContext context) { + + List importRows = context.getImportRows(); + + boolean hasExpUnitObsUnitIDs = importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + + if (hasExpUnitObsUnitIDs) { + long distinctCount = importRows.stream() + .map(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return expRow.getObsUnitID(); + }) + .distinct() + .count(); + + if (distinctCount != importRows.size()) { + // If have ExpUnit ObsUnitIDs and there are duplicates -> Append / Update SubObsUnit Phenotypes + // TODO: different workflow for subobs units? + return appendOverwritePhenotypesWorkflowProvider.get(); + } else { + // If have ExpUnit ObsUnitIDs and all are unique -> Append / Update ExpUnit Phenotypes + return appendOverwritePhenotypesWorkflowProvider.get(); + } + + } else { + // No ObsUnitIDs so creating experiment or appending env + return createNewExperimentWorkflowProvider.get(); + // TODO: different workflow for appending envs? Would have a dependency on DAO to check for existing trial name + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java new file mode 100644 index 000000000..6c0384a0f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/workflow/Workflow.java @@ -0,0 +1,10 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.workflow; + +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; + +public interface Workflow { + + ProcessedData process(ImportContext context); + String getName(); +} From d8ca03202472caa806a89ab141c315cc8c971f1d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 29 Apr 2024 11:57:35 -0400 Subject: [PATCH 02/61] Added trials pending data to get existing step --- .../experiment/create/model/ExistingData.java | 18 -- .../experiment/create/model/PendingData.java | 28 +++ .../steps/GetExistingProcessingStep.java | 193 +++++++++++++++++- .../experiment/create/steps/ProcessStep.java | 6 +- 4 files changed, 218 insertions(+), 27 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java deleted file mode 100644 index 99113cc9f..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ExistingData.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; - -import lombok.*; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; - -import java.util.Map; - -@Getter -@Setter -@Builder -@ToString -@AllArgsConstructor -@NoArgsConstructor -public class ExistingData { - private Map> observationUnitByNameNoScope; - // TODO: add rest of fields -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java new file mode 100644 index 000000000..c3f769a71 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.HashMap; +import java.util.Map; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class PendingData { + private Map> observationUnitByNameNoScope; + private Map> trialByNameNoScope; + private Map> studyByNameNoScope; + private Map> locationByName; + private Map> obsVarDatasetByName; + private Map> existingGermplasmByGID; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index fc69c8105..51997347c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -7,8 +7,12 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -16,7 +20,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -26,29 +30,40 @@ @Prototype @Slf4j -public class GetExistingProcessingStep implements ProcessingStep { +public class GetExistingProcessingStep implements ProcessingStep { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + private final BrAPITrialDAO brAPITrialDAO; + private final BrAPIStudyDAO brAPIStudyDAO; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @Inject - public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + public GetExistingProcessingStep(BrAPIObservationUnitDAO brAPIObservationUnitDAO, + BrAPITrialDAO brAPITrialDAO, + BrAPIStudyDAO brAPIStudyDAO) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + this.brAPITrialDAO = brAPITrialDAO; + this.brAPIStudyDAO = brAPIStudyDAO; } @Override - public ExistingData process(ImportContext input) { + public PendingData process(ImportContext input) { List experimentImportRows = ExperimentUtilities.importRowsToExperimentObservations(input.getImportRows()); Program program = input.getProgram(); - // getExisting + // Populate pending objects with existing status Map> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); + // TODO: populate rest of data - ExistingData existing = ExistingData.builder().observationUnitByNameNoScope(observationUnitByNameNoScope).build(); + PendingData existing = PendingData.builder() + .observationUnitByNameNoScope(observationUnitByNameNoScope) + .trialByNameNoScope(trialByNameNoScope) + .build(); return existing; } @@ -145,4 +160,170 @@ private String createObservationUnitKey(ExperimentObservation importRow) { private String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + + // TODO: also used in other workflow + + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + private void initializeTrialsForExistingObservationUnits(Program program, + Map> observationUnitByNameNoScope, + Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + + + + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java index fa483a7c4..e57523d58 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/ProcessStep.java @@ -2,12 +2,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ExistingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -public class ProcessStep implements ProcessingStep { +public class ProcessStep implements ProcessingStep { @Override - public ProcessedData process(ExistingData input) { + public ProcessedData process(PendingData input) { // TODO: implement return new ProcessedData(); From 77e4a7e6a475b2ad50e6e2a572346d70c8efb15a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:18:24 -0400 Subject: [PATCH 03/61] Addeed studies pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 153 +++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 51997347c..6505d6501 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -3,14 +3,17 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; +import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -35,6 +38,7 @@ public class GetExistingProcessingStep implements ProcessingStep> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); - + Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); // TODO: populate rest of data PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) .trialByNameNoScope(trialByNameNoScope) + .studyByNameNoScope(studyByNameNoScope) .build(); return existing; @@ -323,7 +330,149 @@ private List fetchStudiesByDbId(Set studyDbIds, Program prog return studies; } + /** + * Initializes studies by name without scope. + * + * @param program The program object. + * @param trialByNameNoScope A map of trial names with their corresponding pending import objects. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of study names with their corresponding pending import objects. + * @throws InternalServerException If there is an error while processing the method. + */ + private Map> initializeStudyByNameNoScope(Program program, + Map> trialByNameNoScope, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> studyByName = new HashMap<>(); + if (trialByNameNoScope.size() != 1) { + return studyByName; + } + + try { + initializeStudiesForExistingObservationUnits(program, studyByName, observationUnitByNameNoScope); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); + } + + List existingStudies; + Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + + try { + if (trial.isEmpty()) { + // TODO: throw ValidatorException and return 422 + } + UUID experimentId = trial.get().getId(); + existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + for (BrAPIStudy existingStudy : existingStudies) { + processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); + } + + return studyByName; + } + + /** + * Retrieves the PendingImportObject of a BrAPITrial based on the given list of ExperimentObservation and trialByNameNoScope map. + * + * @param experimentImportRows The list of ExperimentObservation objects. + * @param trialByNameNoScope The map of trial names to PendingImportObject of BrAPITrial. + * @return The Optional containing the PendingImportObject of BrAPITrial, or an empty Optional if no matching trial is found. + */ + private Optional> getTrialPIO(List experimentImportRows, + Map> trialByNameNoScope) { + Optional expTitle = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID()) && StringUtils.isNotBlank(row.getExpTitle())) + .map(ExperimentObservation::getExpTitle) + .findFirst(); + if (expTitle.isEmpty() && trialByNameNoScope.keySet().stream().findFirst().isEmpty()) { + return Optional.empty(); + } + if(expTitle.isEmpty()) { + expTitle = trialByNameNoScope.keySet().stream().findFirst(); + } + return Optional.ofNullable(trialByNameNoScope.get(expTitle.get())); + } + + + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName, + Map> observationUnitByNameNoScope + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + + // TODO: used by both workflows + private PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + // TODO: used by both workflows + private String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } } From 65cf44d33b469bd8964a573788856b676bd7d52c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 29 Apr 2024 17:36:59 -0400 Subject: [PATCH 04/61] Added locations pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 6505d6501..0dcace200 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -25,6 +25,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -39,6 +41,7 @@ public class GetExistingProcessingStep implements ProcessingStep> observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); Map> trialByNameNoScope = initializeTrialByNameNoScope(program, observationUnitByNameNoScope, experimentImportRows); Map> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); + // interesting we're using our data model instead of brapi for locations + Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); // TODO: populate rest of data PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) .trialByNameNoScope(trialByNameNoScope) .studyByNameNoScope(studyByNameNoScope) + .locationByName(locationByName) .build(); return existing; @@ -475,4 +483,51 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { return (yearInt == null) ? "" : yearInt.toString(); } + /** + * Initializes unique location names for a program. + * + * @param program The program object. + * @param studyByNameNoScope A map of study names and corresponding BrAPI study objects. + * @param experimentImportRows A list of experiment observation objects for import. + * @return A map of location names and their corresponding pending import objects. + * @throws InternalServerException If there is an error fetching locations. + */ + private Map> initializeUniqueLocationNames(Program program, + Map> studyByNameNoScope, + List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + } From 8633a3e389cc205c643f336caaba5b7f707403a1 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:37:43 -0400 Subject: [PATCH 05/61] Added dataset pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 78 +++++++++++++++++-- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 0dcace200..ebe0cf5e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -8,14 +8,11 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.core.BrAPISeason; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.*; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; -import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -42,6 +39,7 @@ public class GetExistingProcessingStep implements ProcessingStep> studyByNameNoScope = initializeStudyByNameNoScope(program, trialByNameNoScope, observationUnitByNameNoScope, experimentImportRows); // interesting we're using our data model instead of brapi for locations Map> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); + Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); + // TODO: populate rest of data PendingData existing = PendingData.builder() @@ -78,6 +80,7 @@ public PendingData process(ImportContext input) { .trialByNameNoScope(trialByNameNoScope) .studyByNameNoScope(studyByNameNoScope) .locationByName(locationByName) + .obsVarDatasetByName(obsVarDatasetByName) .build(); return existing; @@ -530,4 +533,63 @@ private Map> initializeUniqueLocati return locationByName; } + /** + * Initializes observation variable dataset by name. + * + * @param program The program associated with the dataset. + * @param trialByNameNoScope The map of trials identified by name without scope. + * @param experimentImportRows The list of experiment observation rows. + * @return The map of observation variable dataset indexed by name. + * + * @throws InternalServerException + */ + private Map> initializeObsVarDatasetByName(Program program, + Map> trialByNameNoScope, + List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows, trialByNameNoScope); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + /** + * Process and cache an object of type BrAPIListDetails. + * + * @param existingList The existing list to be processed and cached + * @param obsVarDatasetByName A map of ObsVarDatasets indexed by name (will be modified in place) + * + * @throws IllegalStateException + */ + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByName.put(existingList.getListName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } + } From ab0364436780552bab8a313f73d875a923962bbe Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 30 Apr 2024 10:56:09 -0400 Subject: [PATCH 06/61] Added germplasm pending data to get existing step --- .../steps/GetExistingProcessingStep.java | 83 ++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index ebe0cf5e9..012a17e77 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -10,6 +10,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; @@ -40,6 +41,7 @@ public class GetExistingProcessingStep implements ProcessingStep> locationByName = initializeUniqueLocationNames(program, studyByNameNoScope, experimentImportRows); Map> obsVarDatasetByName = initializeObsVarDatasetByName(program, trialByNameNoScope, experimentImportRows); - - // TODO: populate rest of data + Map> existingGermplasmByGID = initializeExistingGermplasmByGID(program, observationUnitByNameNoScope, experimentImportRows); PendingData existing = PendingData.builder() .observationUnitByNameNoScope(observationUnitByNameNoScope) @@ -81,6 +84,7 @@ public PendingData process(ImportContext input) { .studyByNameNoScope(studyByNameNoScope) .locationByName(locationByName) .obsVarDatasetByName(obsVarDatasetByName) + .existingGermplasmByGID(existingGermplasmByGID) .build(); return existing; @@ -592,4 +596,77 @@ private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); } + /** + * Initializes existing germplasm objects by germplasm ID (GID). + * + * @param program The program object. + * @param observationUnitByNameNoScope A map of observation unit objects by name. + * @param experimentImportRows A list of experiment observation objects. + * @return A map of existing germplasm objects by germplasm ID. + * + * @throws InternalServerException + */ + private Map> initializeExistingGermplasmByGID(Program program, + Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + /** + * Retrieves a list of germplasm with the specified accession numbers. + * + * @param germplasmAccessionNumbers The list of accession numbers to search for. + * @param programId The ID of the program. + * @return An ArrayList of BrAPIGermplasm objects that match the accession numbers. + * @throws ApiException if there is an error retrieving the germplasm. + */ + private ArrayList getGermplasmByAccessionNumber( + List germplasmAccessionNumbers, + UUID programId) throws ApiException { + List germplasmList = brAPIGermplasmDAO.getGermplasm(programId); + ArrayList resultGermplasm = new ArrayList<>(); + // Search for accession number matches + for (BrAPIGermplasm germplasm : germplasmList) { + for (String accessionNumber : germplasmAccessionNumbers) { + if (germplasm.getAccessionNumber() + .equals(accessionNumber)) { + resultGermplasm.add(germplasm); + break; + } + } + } + return resultGermplasm; + } + } From 9d2e108bbce6beeec72b4571fb5eff96ce750f11 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 1 May 2024 11:31:14 -0400 Subject: [PATCH 07/61] create Middleware class --- .../AppendOverwritePhenotypesWorkflow.java | 17 +++- .../middleware/ExpUnit/ExpUnitMiddleware.java | 8 ++ .../ExpUnit/GetExistingBrAPIData.java | 87 +++++++++++++++++++ .../ExpUnit/ValidateAllRowsHaveIDs.java | 38 ++++++++ .../experiment/middleware/Middleware.java | 34 ++++++++ .../model/ExpImportProcessErrorConstants.java | 24 +++++ .../model/ExpUnitMiddlewareContext.java | 31 +++++++ 7 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 970fc25f3..39e629d36 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,6 +1,10 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; @@ -8,10 +12,19 @@ @Prototype public class AppendOverwritePhenotypesWorkflow implements Workflow { + ExpUnitMiddleware middleware; + GetExistingBrAPIData getExistingBrAPIData; + public AppendOverwritePhenotypesWorkflow(GetExistingBrAPIData getExistingBrAPIData) { + this.middleware.link( + new ValidateAllRowsHaveIDs(), + getExistingBrAPIData + ); + } @Override public ProcessedData process(ImportContext context) { - - // validate that all rows have an ObsUnitID + ExpUnitMiddlewareContext workflowContext = new ExpUnitMiddlewareContext(); + workflowContext.setImportContext(context); + this.middleware.process(workflowContext); // TODO: implement return new ProcessedData(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java new file mode 100644 index 000000000..2cfe6c4a6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java @@ -0,0 +1,8 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; + +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +public abstract class ExpUnitMiddleware extends Middleware { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java new file mode 100644 index 000000000..80c458903 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java @@ -0,0 +1,87 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class GetExistingBrAPIData extends ExpUnitMiddleware { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); + } + + private Map> fetchReferenceObservationUnits( + Set referenceOUIds, + Program program + ) throws ApiException { + Map> pendingUnitById = new HashMap<>(); + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(referenceOUIds), + program + ); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == referenceOUIds.size()) { + // Iterate through reference Observation Units + referenceObsUnits.forEach(unit -> { + // Get external reference for the Observation Unit + BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); + + // Set pending Observation Unit by its ID + pendingUnitById.put( + unitXref.getReferenceId(), + new PendingImportObject( + ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) + ); + }); + } else { + // Handle missing Observation Unit IDs + List missingIds = new ArrayList<>(referenceOUIds); + Set fetchedIds = referenceObsUnits.stream().map(unit -> + Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) + .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) + .getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + } + + return pendingUnitById; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new ApiException(e); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java new file mode 100644 index 000000000..26c9f3466 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java @@ -0,0 +1,38 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; + +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import java.util.HashSet; +import java.util.Set; + +public class ValidateAllRowsHaveIDs extends ExpUnitMiddleware { + private boolean hasAllReferenceUnitIds = true; + private boolean hasNoReferenceUnitIds = true; + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + context.setReferenceOUIds(collateReferenceOUIds(context)); + return processNext(context); + } + private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { + Set referenceOUIds = new HashSet<>(); + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + return referenceOUIds; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java new file mode 100644 index 000000000..f9dfb31e6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -0,0 +1,34 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; + +public abstract class Middleware { + + Middleware next; + + /** + * Builds chains of middleware objects. + */ + public static Middleware link(Middleware first, Middleware... chain) { + Middleware head = first; + for (Middleware nextInChain: chain) { + head.next = nextInChain; + head = nextInChain; + } + return first; + } + + /** + * Subclasses will implement this method with processing steps. + */ + public abstract boolean process(T context); + + /** + * Runs check on the next object in chain or ends traversing if we're in + * last object in chain. + */ + protected boolean processNext(T context) { + if (next == null) { + return true; + } + return next.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java new file mode 100644 index 000000000..8509d74f6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java @@ -0,0 +1,24 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import com.fasterxml.jackson.annotation.JsonValue; + +public class ExpImportProcessErrorConstants { + + public static final CharSequence COMMA_DELIMITER = ","; + public enum ExpImportProcessErrMessage { + + MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), + PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + private String value; + + ExpImportProcessErrMessage(String value) { + this.value = value; + } + + @JsonValue + public String getValue() { + return value; + } + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java new file mode 100644 index 000000000..2ce05216b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -0,0 +1,31 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.*; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class ExpUnitMiddlewareContext { + @Getter + @Setter + private ImportContext importContext; + @Getter + @Setter + private Set referenceOUIds = new HashSet<>(); + private Map> pendingTrialByOUId = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); + private Map> pendingObsUnitByOUId = new HashMap<>(); + private Map> pendingObsDatasetByOUId = new HashMap<>(); + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); + +} From d97f27c41cc59e6d6cf2c09db97c8630a6430f1f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 1 May 2024 20:23:04 -0400 Subject: [PATCH 08/61] add comensation methods to Middleware class --- .../experiment/middleware/Middleware.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index f9dfb31e6..d0f9f06c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -3,6 +3,7 @@ public abstract class Middleware { Middleware next; + Middleware prior; /** * Builds chains of middleware objects. @@ -10,6 +11,7 @@ public abstract class Middleware { public static Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { + nextInChain.prior = head; head.next = nextInChain; head = nextInChain; } @@ -17,13 +19,16 @@ public static Middleware link(Middleware first, Middleware... chain) { } /** - * Subclasses will implement this method with processing steps. + * Subclasses will implement this local transaction. */ public abstract boolean process(T context); - /** - * Runs check on the next object in chain or ends traversing if we're in - * last object in chain. + * Subclasses will implement this method to handle errors and possibly undo the local transaction. + */ + public abstract boolean compensate(T context); + /** + * Processes the next local transaction or ends traversing if we're at the + * last local transaction of the transaction. */ protected boolean processNext(T context) { if (next == null) { @@ -31,4 +36,15 @@ protected boolean processNext(T context) { } return next.process(context); } + + /** + * Runs the compensating local transaction for the prior local transaction or ends traversing if + * we're at the first local transaction of the transaction. + */ + protected boolean compensatePrior(T context) { + if (prior == null) { + return true; + } + return prior.compensate(context); + } } From 3a0481fea4c43422bfd750b5ca587b681c1b4bac Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 09:36:28 -0400 Subject: [PATCH 09/61] add compensate method to middleware --- .../ExpUnit/GetExistingBrAPIData.java | 77 +++++++++++++------ .../ExpUnit/ValidateAllRowsHaveIDs.java | 17 +++- .../experiment/middleware/Middleware.java | 8 +- .../experiment/model/MiddlewareError.java | 21 +++++ 4 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java index 80c458903..01ae96ef5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java @@ -14,6 +14,7 @@ import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -34,54 +35,86 @@ public GetExistingBrAPIData(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { @Override public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); } + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // handle the error in the prior local transaction + return compensatePrior(context, error); + } private Map> fetchReferenceObservationUnits( - Set referenceOUIds, - Program program - ) throws ApiException { + ExpUnitMiddlewareContext context) { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(referenceOUIds), - program + new ArrayList(context.getReferenceOUIds()), + context.getImportContext().getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == referenceOUIds.size()) { + if (referenceObsUnits.size() == context.getReferenceOUIds().size()) { + // Iterate through reference Observation Units - referenceObsUnits.forEach(unit -> { - // Get external reference for the Observation Unit - BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) - .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); - - // Set pending Observation Unit by its ID - pendingUnitById.put( - unitXref.getReferenceId(), - new PendingImportObject( - ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) + for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + + // Set pending Observation Unit by its ID + pendingUnitById.put( + xref.getReferenceId(), + new PendingImportObject<>( + ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) + ); + }, + () -> { + + // but throw an error if no unit ID + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + })); + } ); - }); + + + } } else { - // Handle missing Observation Unit IDs - List missingIds = new ArrayList<>(referenceOUIds); + // Handle case of missing Observation Units in data store + List missingIds = new ArrayList<>(context.getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream().map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) .getReferenceId()) .collect(Collectors.toSet()); missingIds.removeAll(fetchedIds); - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + + // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + })); } return pendingUnitById; } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new ApiException(e); + + // throw an error if problem getting data from the brapi data store + this.compensate(context, new MiddlewareError(() -> { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + try { + throw new ApiException(e); + } catch (ApiException ex) { + throw new RuntimeException(ex); + } + })); } + return pendingUnitById; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java index 26c9f3466..32219a821 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java @@ -3,6 +3,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import java.util.HashSet; import java.util.Set; @@ -16,6 +17,16 @@ public boolean process(ExpUnitMiddlewareContext context) { context.setReferenceOUIds(collateReferenceOUIds(context)); return processNext(context); } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + + // undo the prior local transaction + return compensatePrior(context, error); + } + private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { Set referenceOUIds = new HashSet<>(); for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { @@ -25,8 +36,12 @@ private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { hasAllReferenceUnitIds = false; } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + this.compensate(context, new MiddlewareError(()->{ + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + })); + } else { // Add ObsUnitID to referenceOUIds referenceOUIds.add(importRow.getObsUnitID()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index d0f9f06c2..d2d7443b8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -1,5 +1,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.middleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + public abstract class Middleware { Middleware next; @@ -25,7 +27,7 @@ public static Middleware link(Middleware first, Middleware... chain) { /** * Subclasses will implement this method to handle errors and possibly undo the local transaction. */ - public abstract boolean compensate(T context); + public abstract boolean compensate(T context, MiddlewareError error); /** * Processes the next local transaction or ends traversing if we're at the * last local transaction of the transaction. @@ -41,10 +43,10 @@ protected boolean processNext(T context) { * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. */ - protected boolean compensatePrior(T context) { + protected boolean compensatePrior(T context, MiddlewareError error) { if (prior == null) { return true; } - return prior.compensate(context); + return prior.compensate(context, error); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java new file mode 100644 index 000000000..49960ae3a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/MiddlewareError.java @@ -0,0 +1,21 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.model; + +import lombok.Getter; +import lombok.Setter; + +public class MiddlewareError { + @Getter + @Setter + String localTransactionName; + Runnable handler; + + public MiddlewareError(Runnable handler) { + this.handler = handler; + } + + public void tag(String name) { + if (this.getLocalTransactionName() == null) { + this.setLocalTransactionName(name); + } + } +} From 2741485c7911eadf2811f6ef439fef0bbe913859 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 May 2024 13:33:07 -0400 Subject: [PATCH 10/61] Work on breaking out common workflow code --- .../experiment/ExperimentUtilities.java | 2 + .../steps/GetExistingProcessingStep.java | 231 +----------------- .../experiment/services/StudyService.java | 120 +++++++++ .../experiment/services/TrialService.java | 182 ++++++++++++++ 4 files changed, 315 insertions(+), 220 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index bd90ee222..e0bdfe432 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -9,6 +9,7 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() @@ -17,4 +18,5 @@ public static List importRowsToExperimentObservations(Lis } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index 012a17e77..d01324e4f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -22,6 +22,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; @@ -38,10 +40,11 @@ public class GetExistingProcessingStep implements ProcessingStep> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, - List experimentImportRows) { - Map> trialByName = new HashMap<>(); - - initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); - - List uniqueTrialNames = experimentImportRows.stream() - .filter(row -> StringUtils.isBlank(row.getObsUnitID())) - .map(ExperimentObservation::getExpTitle) - .distinct() - .collect(Collectors.toList()); - try { - brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialByName) - ); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - - return trialByName; - } - - // TODO: also used in other workflow - - /** - * Initializes trials for existing observation units. - * - * @param program The program object. - * @param observationUnitByNameNoScope A map containing observation units by name (without scope). - * @param trialByName A map containing trials by name. (will be modified in place) - * - */ - private void initializeTrialsForExistingObservationUnits(Program program, - Map> observationUnitByNameNoScope, - Map> trialByName) { - if(observationUnitByNameNoScope.size() > 0) { - Set trialDbIds = new HashSet<>(); - Set studyDbIds = new HashSet<>(); - - observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); - - //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId - if(!studyDbIds.isEmpty()) { - try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - try { - List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); - if (trials.size() != trialDbIds.size()) { - List missingIds = new ArrayList<>(trialDbIds); - missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); - throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - - trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - } - - /** - * This method processes an existing trial, retrieves the experiment ID from the trial's external references, - * and caches the trial with the corresponding experiment ID in a map. - * - * @param existingTrial The existing BrAPITrial object to be processed and cached. - * @param program The Program object associated with the trial. - * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) - * - * @throws InternalServerException - */ - private void processAndCacheTrial( - BrAPITrial existingTrial, - Program program, - Map> trialByNameNoScope) { - - //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - - trialByNameNoScope.put( - Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); - } - - /** - * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. - * - * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. - * @param program The program associated with the studies. - * @return A set of trial DbIds corresponding to the provided study DbIds. - * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. - * @throws IllegalStateException If the trial DbId is not set for an existing study. - */ - private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { - Set trialDbIds = new HashSet<>(); - List studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> { - if (StringUtils.isBlank(study.getTrialDbId())) { - throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); - } - trialDbIds.add(study.getTrialDbId()); - }); - - return trialDbIds; - } - - /** - * Fetches a list of BrAPI studies by their study database IDs for a given program. - * - * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. - * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. - * - * @param studyDbIds a Set of Strings representing the study database IDs to fetch - * @param program the Program object representing the program context in which to fetch studies - * @return a List of BrAPIStudy objects matching the provided study database IDs - * - * @throws ApiException if there is an issue fetching the studies - * @throws IllegalStateException if any requested study database IDs are not found in the result set - */ - private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { - List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if (studies.size() != studyDbIds.size()) { - List missingIds = new ArrayList<>(studyDbIds); - missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException( - "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); - } - return studies; - } - /** * Initializes studies by name without scope. * @@ -383,7 +226,7 @@ private Map> initializeStudyByNameNoScop UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); for (BrAPIStudy existingStudy : existingStudies) { - processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + studyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); } } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); @@ -434,60 +277,8 @@ private void initializeStudiesForExistingObservationUnits( List studies = fetchStudiesByDbId(studyDbIds, program); for (BrAPIStudy study : studies) { - processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); - } - } - - // TODO: used by both workflows - private PendingImportObject processAndCacheStudy( - BrAPIStudy existingStudy, - Program program, - Function getterFunction, - Map> studyMap) throws Exception { - PendingImportObject pendingStudy; - BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); - // map season dbid to year - String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season - if(StringUtils.isNotBlank(seasonDbId)) { - String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); - existingStudy.setSeasons(Collections.singletonList(seasonYear)); - } - pendingStudy = new PendingImportObject<>( - ImportObjectState.EXISTING, - (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), - UUID.fromString(xref.getReferenceId()) - ); - studyMap.put( - Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), - pendingStudy - ); - return pendingStudy; - } - - // TODO: used by both workflows - private String seasonDbIdToYear(String seasonDbId, UUID programId) { - String year = null; - // TODO: add season objects to redis cache then just extract year from those - // removing this for now here - //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible - // year = this.seasonDbIdToYearCache.get(seasonDbId); - //} else { - year = seasonDbIdToYearFromDatabase(seasonDbId, programId); - // this.seasonDbIdToYearCache.put(seasonDbId, year); - //} - return year; - } - - private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason season = null; - try { - season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); + studyService.processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); } - Integer yearInt = (season == null) ? null : season.getYear(); - return (yearInt == null) ? "" : yearInt.toString(); } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java new file mode 100644 index 000000000..daca5630f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -0,0 +1,120 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.reactivex.functions.Function; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPISeason; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class StudyService { + + private final BrAPISeasonDAO brAPISeasonDAO; + private final BrAPIStudyDAO brAPIStudyDAO; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public StudyService(BrAPISeasonDAO brAPISeasonDAO, + BrAPIStudyDAO brAPIStudyDAO) { + this.brAPISeasonDAO = brAPISeasonDAO; + this.brAPIStudyDAO = brAPIStudyDAO; + } + + // TODO: used by both workflows + public PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap) throws Exception { + PendingImportObject pendingStudy; + BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; + } + + // TODO: used by both workflows + private String seasonDbIdToYear(String seasonDbId, UUID programId) { + String year = null; + // TODO: add season objects to redis cache then just extract year from those + // removing this for now here + //if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible + // year = this.seasonDbIdToYearCache.get(seasonDbId); + //} else { + year = seasonDbIdToYearFromDatabase(seasonDbId, programId); + // this.seasonDbIdToYearCache.put(seasonDbId, year); + //} + return year; + } + + // TODO: used by both workflows + private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { + BrAPISeason season = null; + try { + season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + } + Integer yearInt = (season == null) ? null : season.getYear(); + return (yearInt == null) ? "" : yearInt.toString(); + } + + /** + * Fetches a list of BrAPI studies by their study database IDs for a given program. + * + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set + */ + private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); + if (studies.size() != studyDbIds.size()) { + List missingIds = new ArrayList<>(studyDbIds); + missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + return studies; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java new file mode 100644 index 000000000..76e365cca --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -0,0 +1,182 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class TrialService { + private final BrAPITrialDAO brAPITrialDAO; + + private final StudyService studyService; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public TrialService(BrAPITrialDAO brAPITrialDAO, + StudyService studyService) { + this.brAPITrialDAO = brAPITrialDAO; + this.studyService = studyService; + } + + // TODO: also used in other workflow + + /** + * Initializes trials for existing observation units. + * + * @param program The program object. + * @param observationUnitByNameNoScope A map containing observation units by name (without scope). + * @param trialByName A map containing trials by name. (will be modified in place) + * + */ + public void initializeTrialsForExistingObservationUnits(Program program, + Map> observationUnitByNameNoScope, + Map> trialByName) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + + /** + * Fetches trial DbIds for the given study DbIds by using the BrAPI studies API. + * + * @param studyDbIds The set of study DbIds for which to fetch trial DbIds. + * @param program The program associated with the studies. + * @return A set of trial DbIds corresponding to the provided study DbIds. + * @throws ApiException If there was an error while fetching the studies or if a study does not have a trial DbId. + * @throws IllegalStateException If the trial DbId is not set for an existing study. + */ + private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { + Set trialDbIds = new HashSet<>(); + List studies = fetchStudiesByDbId(studyDbIds, program); + studies.forEach(study -> { + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbIds.add(study.getTrialDbId()); + }); + + return trialDbIds; + } + + /** + * This method processes an existing trial, retrieves the experiment ID from the trial's external references, + * and caches the trial with the corresponding experiment ID in a map. + * + * @param existingTrial The existing BrAPITrial object to be processed and cached. + * @param program The Program object associated with the trial. + * @param trialByNameNoScope The map to cache the trial by its name without program scope. (will be modified in place) + * + * @throws InternalServerException + */ + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + Map> trialByNameNoScope) { + + //get TrialId from existingTrial + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + trialByNameNoScope.put( + Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Initializes trials by name without scope for the given program. + * + * @param program the program to initialize trials for + * @param observationUnitByNameNoScope a map of observation units by name without scope + * @param experimentImportRows a list of experiment observation rows + * @return a map of trials by name with pending import objects + * + * @throws InternalServerException + */ + private Map> initializeTrialByNameNoScope(Program program, Map> observationUnitByNameNoScope, + List experimentImportRows) { + Map> trialByName = new HashMap<>(); + + initializeTrialsForExistingObservationUnits(program, observationUnitByNameNoScope, trialByName); + + List uniqueTrialNames = experimentImportRows.stream() + .filter(row -> StringUtils.isBlank(row.getObsUnitID())) + .map(ExperimentObservation::getExpTitle) + .distinct() + .collect(Collectors.toList()); + try { + brAPITrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialByName) + ); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + return trialByName; + } + +} From 8f761aea7d15bc4ef2d150a93b3c818a59532ecb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 14:48:32 -0400 Subject: [PATCH 11/61] create ExpUnitContextService --- .../AppendOverwritePhenotypesWorkflow.java | 17 ++-- .../ExpUnitContextService.java | 96 +++++++++++++++++++ .../middleware}/ExpUnitMiddleware.java | 2 +- .../middleware}/GetExistingBrAPIData.java | 12 +-- .../middleware}/ValidateAllRowsHaveIDs.java | 4 +- .../appendoverwrite/model/ExpUnitContext.java | 29 ++++++ .../model/ExpUnitMiddlewareContext.java | 29 +----- 7 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{middleware/ExpUnit => appendoverwrite/middleware}/ExpUnitMiddleware.java (90%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{middleware/ExpUnit => appendoverwrite/middleware}/GetExistingBrAPIData.java (93%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{middleware/ExpUnit => appendoverwrite/middleware}/ValidateAllRowsHaveIDs.java (93%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 39e629d36..fe2e1ae3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -1,23 +1,28 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.GetExistingBrAPIData; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; import org.breedinginsight.brapps.importer.services.processors.experiment.workflow.Workflow; +import javax.inject.Inject; +import javax.inject.Provider; + + @Prototype public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; - GetExistingBrAPIData getExistingBrAPIData; - public AppendOverwritePhenotypesWorkflow(GetExistingBrAPIData getExistingBrAPIData) { + Provider getExistingBrAPIDataProvider; + @Inject + public AppendOverwritePhenotypesWorkflow(Provider getExistingBrAPIDataProvider) { this.middleware.link( new ValidateAllRowsHaveIDs(), - getExistingBrAPIData + getExistingBrAPIDataProvider.get() ); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java new file mode 100644 index 000000000..39a5e0215 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -0,0 +1,96 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class ExpUnitContextService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + /** + * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. + * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. + * + * @param referenceOUIds A set of reference Observation Unit IDs to retrieve + * @param program The Program associated with the Observation Units + * @return A Map containing pending Observation Units by their ID + * @throws ApiException if an error occurs during the process + */ + public Map> fetchReferenceObservationUnits( + Set referenceOUIds, + Program program + ) throws ApiException { + Map> pendingUnitById = new HashMap<>(); + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(referenceOUIds), + program + ); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == referenceOUIds.size()) { + for (BrAPIObservationUnit unit : referenceObsUnits) {// Iterate through reference Observation Units + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + + // Set pending Observation Unit by its ID + pendingUnitById.put( + xref.getReferenceId(), + new PendingImportObject<>( + ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) + ); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + } + } else {// Handle case of missing Observation Units in data store + List missingIds = new ArrayList<>(referenceOUIds); + Set fetchedIds = referenceObsUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + } + + return pendingUnitById; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new ApiException(e); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java similarity index 90% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 2cfe6c4a6..81e4e1a0d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index 01ae96ef5..e87583d6b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; @@ -7,15 +7,13 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -53,14 +51,14 @@ private Map> fetchReferenceObs try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(context.getReferenceOUIds()), + new ArrayList(context.getExpUnitContext().getReferenceOUIds()), context.getImportContext().getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == context.getReferenceOUIds().size()) { + if (referenceObsUnits.size() == context.getExpUnitContext().getReferenceOUIds().size()) { // Iterate through reference Observation Units for (BrAPIObservationUnit unit : referenceObsUnits) {// Get external reference for the Observation Unit @@ -88,7 +86,7 @@ private Map> fetchReferenceObs } } else { // Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(context.getReferenceOUIds()); + List missingIds = new ArrayList<>(context.getExpUnitContext().getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream().map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java similarity index 93% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java index 32219a821..45c728453 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/ExpUnit/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java @@ -1,7 +1,7 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.middleware.ExpUnit; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java new file mode 100644 index 000000000..50ca1f4d2 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -0,0 +1,29 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model; + +import lombok.Getter; +import lombok.Setter; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.ProgramLocation; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Getter +@Setter +public class ExpUnitContext { + private Set referenceOUIds = new HashSet<>(); + private Map> pendingTrialByOUId = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); + private Map> pendingObsUnitByOUId = new HashMap<>(); + private Map> pendingObsDatasetByOUId = new HashMap<>(); + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 2ce05216b..6a8f9183e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -1,31 +1,12 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import lombok.*; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.model.ProgramLocation; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +@Getter +@Setter public class ExpUnitMiddlewareContext { - @Getter - @Setter - private ImportContext importContext; - @Getter - @Setter - private Set referenceOUIds = new HashSet<>(); - private Map> pendingTrialByOUId = new HashMap<>(); - private Map> pendingStudyByOUId = new HashMap<>(); - private Map> pendingObsUnitByOUId = new HashMap<>(); - private Map> pendingObsDatasetByOUId = new HashMap<>(); - private Map> pendingLocationByOUId = new HashMap<>(); - private Map> pendingGermplasmByOUId = new HashMap<>(); + private ImportContext importContext; + private ExpUnitContext expUnitContext; } From 8fe5fe29d4379eebf2aa649d90ab91948bfef160 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 16:04:19 -0400 Subject: [PATCH 12/61] add mapPendingOUsByName service method --- .../experiment/ExperimentUtilities.java | 6 +++ .../ExpUnitContextService.java | 44 ++++++++++++------- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index e0bdfe432..92d333ce9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -17,6 +17,12 @@ public static List importRowsToExperimentObservations(Lis .collect(Collectors.toList()); } + public static String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + public static String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 39a5e0215..85a415717 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -10,7 +10,10 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -29,31 +32,23 @@ public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; } - /** - * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. - * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. - * - * @param referenceOUIds A set of reference Observation Unit IDs to retrieve - * @param program The Program associated with the Observation Units - * @return A Map containing pending Observation Units by their ID - * @throws ApiException if an error occurs during the process - */ + public Map> fetchReferenceObservationUnits( - Set referenceOUIds, - Program program + ImportContext importContext, + ExpUnitContext expUnitContext ) throws ApiException { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(referenceOUIds), - program + new ArrayList(expUnitContext.getReferenceOUIds()), + importContext.getProgram() ); // Construct the DeltaBreed observation unit source for external references String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - if (referenceObsUnits.size() == referenceOUIds.size()) { + if (referenceObsUnits.size() == expUnitContext.getReferenceOUIds().size()) { for (BrAPIObservationUnit unit : referenceObsUnits) {// Iterate through reference Observation Units // Get external reference for the Observation Unit @@ -76,7 +71,7 @@ public Map> fetchReferenceObse ); } } else {// Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(referenceOUIds); + List missingIds = new ArrayList<>(expUnitContext.getReferenceOUIds()); Set fetchedIds = referenceObsUnits.stream() .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) @@ -93,4 +88,23 @@ public Map> fetchReferenceObse throw new ApiException(e); } } + + private Map> mapPendingObservationUnitByName( + ExpUnitContext expUnitContext, + ImportContext importContext + ) { + Map> pendingUnitByName = new HashMap<>(); + for (Map.Entry> entry : expUnitContext.getPendingObsUnitByOUId().entrySet()) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getStudyName(), + importContext.getProgram().getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getObservationUnitName(), + importContext.getProgram().getKey() + ); + pendingUnitByName.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), entry.getValue()); + } + return pendingUnitByName; + } } From 6be417ab321082fe852e848d37636e4d482fbd7e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 2 May 2024 16:57:58 -0400 Subject: [PATCH 13/61] add common methods --- .../model/ExpUnitMiddlewareContext.java | 2 ++ .../experiment/model/ProcessedData.java | 4 ++++ .../experiment/services/StudyService.java | 19 ++++++++++++++++++- .../experiment/services/TrialService.java | 19 ++++++++++--------- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 6a8f9183e..740707142 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -2,6 +2,7 @@ import lombok.*; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @Getter @Setter @@ -9,4 +10,5 @@ public class ExpUnitMiddlewareContext { private ImportContext importContext; private ExpUnitContext expUnitContext; + private PendingData pendingData; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java index 4c8fe464d..551af741b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -1,6 +1,9 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; @Getter @Setter @@ -8,4 +11,5 @@ @ToString @NoArgsConstructor public class ProcessedData { + Map statistics; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index daca5630f..e3effb700 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -106,7 +106,7 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { * @throws ApiException if there is an issue fetching the studies * @throws IllegalStateException if any requested study database IDs are not found in the result set */ - private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + public List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); if (studies.size() != studyDbIds.size()) { List missingIds = new ArrayList<>(studyDbIds); @@ -117,4 +117,21 @@ private List fetchStudiesByDbId(Set studyDbIds, Program prog return studies; } + // TODO: used by both workflows + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName + ) throws Exception { + Set studyDbIds = observationUnitByNameNoScope.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); + + List studies = fetchStudiesByDbId(studyDbIds, program); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 76e365cca..d4dbc3c71 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -16,6 +16,8 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -51,14 +53,13 @@ public TrialService(BrAPITrialDAO brAPITrialDAO, * @param trialByName A map containing trials by name. (will be modified in place) * */ - public void initializeTrialsForExistingObservationUnits(Program program, - Map> observationUnitByNameNoScope, - Map> trialByName) { - if(observationUnitByNameNoScope.size() > 0) { + public void initializeTrialsForExistingObservationUnits(ImportContext importContext, + PendingData pendingData) { + if(pendingData.getObservationUnitByNameNoScope().size() > 0) { Set trialDbIds = new HashSet<>(); Set studyDbIds = new HashSet<>(); - observationUnitByNameNoScope.values() + pendingData.getObservationUnitByNameNoScope().values() .forEach(pio -> { BrAPIObservationUnit existingOu = pio.getBrAPIObject(); if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { @@ -75,7 +76,7 @@ public void initializeTrialsForExistingObservationUnits(Program program, //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId if(!studyDbIds.isEmpty()) { try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, importContext.getProgram())); } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -83,14 +84,14 @@ public void initializeTrialsForExistingObservationUnits(Program program, } try { - List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, importContext.getProgram()); if (trials.size() != trialDbIds.size()) { List missingIds = new ArrayList<>(trialDbIds); missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); } - trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); + trials.forEach(trial -> processAndCacheTrial(trial, importContext.getProgram(), pendingData.getTrialByNameNoScope())); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -109,7 +110,7 @@ public void initializeTrialsForExistingObservationUnits(Program program, */ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program program) throws ApiException { Set trialDbIds = new HashSet<>(); - List studies = fetchStudiesByDbId(studyDbIds, program); + List studies = studyService.fetchStudiesByDbId(studyDbIds, program); studies.forEach(study -> { if (StringUtils.isBlank(study.getTrialDbId())) { throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); From 82d02e61b16eb28535c94922d8d7e8d960e4559e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 09:40:08 -0400 Subject: [PATCH 14/61] add service methods for getting existing brapi data --- .../experiment/services/DatasetService.java | 98 ++++++++++++++++ .../experiment/services/GermplasmService.java | 92 +++++++++++++++ .../experiment/services/LocationService.java | 110 ++++++++++++++++++ .../experiment/services/StudyService.java | 18 +++ .../experiment/services/TrialService.java | 25 ++++ 5 files changed, 343 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java new file mode 100644 index 000000000..2b5a75aad --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -0,0 +1,98 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; +import org.brapi.v2.model.core.BrAPIListTypes; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; + +public class DatasetService { + // TODO: used by expunit worflow + public Map> initializeObsVarDatasetForExistingObservationUnits( + Map> trialByName, + Program program) { + Map> obsVarDatasetByName = new HashMap<>(); + + if (trialByName.size() > 0 && + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialByName.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + // TODO: used by create workflow + public Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { + Map> obsVarDatasetByName = new HashMap<>(); + + Optional> trialPIO = getTrialPIO(experimentImportRows); + + if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialPIO.get().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByName; + } + + // TODO: used by expunit workflow + public Map> mapPendingObsDatasetByOUId( + String unitId, + Map> trialByOUId, + Map> obsVarDatasetByName, + Map> obsVarDatasetByOUId) { + if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && + trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); + } + + return obsVarDatasetByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java new file mode 100644 index 000000000..c15e71fd9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java @@ -0,0 +1,92 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; +import java.util.stream.Collectors; + +public class GermplasmService { + // TODO: used by expunit workflow + public Map> initializeGermplasmByGIDForExistingObservationUnits( + Map> unitByName, + Program program) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(unitByName.size() > 0) { + Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + // TODO: used by create worflow + public Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(observationUnitByNameNoScope.size() > 0) { + Set germplasmDbIds = observationUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueGermplasmGIDs = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getGid) + .distinct() + .collect(Collectors.toList()); + + try { + existingGermplasms.addAll(this.getGermplasmByAccessionNumber(uniqueGermplasmGIDs, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); + }); + return existingGermplasmByGID; + } + + // TODO: used by expunit workflow + public Map> mapGermplasmByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> germplasmByName, + Map> germplasmByOUId) { + String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); + germplasmByOUId.put(unitId, germplasmByName.get(gid)); + + return germplasmByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java new file mode 100644 index 000000000..174150992 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java @@ -0,0 +1,110 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Singleton +@Slf4j +public class LocationService { + + // used by expunit workflow + public Map> initializeLocationByName( + Program program, + Map> studyByName) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByName.size() > 0) { + Set locationDbIds = studyByName.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + existingLocations.forEach(existingLocation -> locationByName.put( + existingLocation.getName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) + ) + ); + return locationByName; + } + + // TODO: used by create workflow + public Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByNameNoScope.size() > 0) { + Set locationDbIds = studyByNameNoScope.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + List uniqueLocationNames = experimentImportRows.stream() + .filter(experimentObservation -> StringUtils.isBlank(experimentObservation.getObsUnitID())) + .map(ExperimentObservation::getEnvLocation) + .distinct() + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + try { + existingLocations.addAll(locationService.getLocationsByName(uniqueLocationNames, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + + existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); + return locationByName; + } + + // TODO: used by expunit workflow + public Map> mapPendingLocationByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByOUId, + Map> locationByName, + Map> locationByOUId + ) { + if (unit.getLocationName() != null) { + locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); + } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { + locationByOUId.put( + unitId, + locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) + ); + } else { + throw new IllegalStateException("Observation unit missing location: " + unitId); + } + + return locationByOUId; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index e3effb700..b69e805c8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -9,6 +9,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -134,4 +135,21 @@ private void initializeStudiesForExistingObservationUnits( } } + // TODO: used by expunit workflow + public Map> mapPendingStudyByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByName, + Map> studyByOUId, + Program program + ) { + if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + studyByOUId.put(unitId, studyByName.get(studyName)); + } else { + throw new IllegalStateException("Observation unit missing study name: " + unitId); + } + + return studyByOUId; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index d4dbc3c71..28f536081 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -180,4 +180,29 @@ private Map> initializeTrialByNameNoScop return trialByName; } + // TODO: used by expunit workflow + public Map> mapPendingTrialByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> trialByName, + Map> studyByName, + Map> trialByOUId, + Program program + ) { + String trialName; + if (unit.getTrialName() != null) { + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); + } else if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( + studyByName.get(studyName).getBrAPIObject().getTrialName(), + program.getKey() + ); + } else { + throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); + } + trialByOUId.put(unitId, trialByName.get(trialName)); + + return trialByOUId; + } } From 1ca976d9abec83e50d42c9a1f4e8e47d8d98229b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 09:49:39 -0400 Subject: [PATCH 15/61] add overloaded trial methods for fetching trial pio --- .../experiment/services/TrialService.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 28f536081..6c07df8f7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -16,14 +16,19 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; +import java.math.BigInteger; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; @Singleton @@ -205,4 +210,52 @@ public Map> mapPendingTrialByOUId( return trialByOUId; } + + // TODO: overloaded method used by expunit workflow + public PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext, + ExpUnitContext expUnitContext + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + + + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + + + return trialPio; + } + + // TODO: overloaded method used by create workflow + private PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext + ) throws UnprocessableEntityException { + PendingImportObject trialPio; + + + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + } + + return trialPio; + } } From b4906a54fbaaa790dfcbad3b1b768c9afaad5b6f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 10:08:10 -0400 Subject: [PATCH 16/61] add pending data context to signature --- .../experiment/ExperimentUtilities.java | 16 ++++++++++++++++ .../experiment/services/TrialService.java | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 92d333ce9..1c55f7545 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -2,8 +2,10 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class ExperimentUtilities { @@ -25,4 +27,18 @@ public static String createObservationUnitKey(String studyName, String obsUnitNa return studyName + obsUnitName; } + /** + * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. + * + * @param map The map from which to retrieve the single value. + * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. + * @return The single value from the map. + * @throws UnprocessableEntityException if the map does not contain exactly one entry. + */ + public V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { + if (map.size() != 1) { + throw new UnprocessableEntityException(message); + } + return map.values().iterator().next(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 6c07df8f7..83b36f9d7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -214,6 +214,7 @@ public Map> mapPendingTrialByOUId( // TODO: overloaded method used by expunit workflow public PendingImportObject fetchOrCreateTrialPIO( ImportContext importContext, + PendingData pendingData, ExpUnitContext expUnitContext ) throws UnprocessableEntityException { PendingImportObject trialPio; @@ -227,8 +228,9 @@ public PendingImportObject fetchOrCreateTrialPIO( } // TODO: overloaded method used by create workflow - private PendingImportObject fetchOrCreateTrialPIO( - ImportContext importContext + public PendingImportObject fetchOrCreateTrialPIO( + ImportContext importContext, + PendingData pendingData ) throws UnprocessableEntityException { PendingImportObject trialPio; From 8f15d08c985a650b0c09ef045ed76ce5e115b6b8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 11:04:37 -0400 Subject: [PATCH 17/61] add ValidateService --- .../experiment/ExperimentUtilities.java | 18 +++ .../experiment/services/DatasetService.java | 90 +++++++++++++ .../experiment/services/LocationService.java | 26 ++++ .../services/ObservationUnitService.java | 127 ++++++++++++++++++ .../experiment/services/StudyService.java | 102 ++++++++++++++ .../experiment/services/ValidateService.java | 112 +++++++++++++++ 6 files changed, 475 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 1c55f7545..25b87a0e8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,7 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment; +import com.google.gson.JsonObject; +import org.brapi.v2.model.core.BrAPIStudy; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import java.util.List; @@ -41,4 +45,18 @@ public V getSingleEntryValue(Map map, String message) throws Unproc } return map.values().iterator().next(); } + + /* + * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) + * */ + public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, String year) { + JsonObject additionalInfo = study.getAdditionalInfo(); + if (additionalInfo==null){ + additionalInfo = new JsonObject(); + study.setAdditionalInfo(additionalInfo); + } + if( additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { + additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index 2b5a75aad..db5fcdedb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -8,9 +8,15 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import java.util.*; @@ -95,4 +101,88 @@ public Map> mapPendingObsDatasetBy return obsVarDatasetByOUId; } + + // TODO: used by both workflows + public void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { + BrAPIListDetails details = pio.getBrAPIObject(); + referencedTraits.forEach(trait -> { + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + + // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { + details.getData().add(id); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING == pio.getState()) { + details.getData().add(id); + pio.setState(ImportObjectState.MUTATED); + } + }); + } + + // TODO: used by expunit workflow + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + PendingImportObject trialPIO = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } + + // TODO: used by create workflow + public void fetchOrCreateDatasetPIO(ImportContext importContext, + PendingData pendingData, + List referencedTraits) throws UnprocessableEntityException { + PendingImportObject pio; + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + + String name = String.format("Observation Dataset [%s-%s]", + program.getKey(), + trialPIO.getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER) + .getAsString()); + if (obsVarDatasetByName.containsKey(name)) { + pio = obsVarDatasetByName.get(name); + } else { + UUID id = UUID.randomUUID(); + BrAPIListDetails newDataset = importRow.constructDatasetDetails( + name, + id, + BRAPI_REFERENCE_SOURCE, + program, + trialPIO.getId().toString()); + pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + if (ImportObjectState.EXISTING == trialPIO.getState()) { + trialPIO.setState(ImportObjectState.MUTATED); + } + obsVarDatasetByName.put(name, pio); + } + addObsVarsToDatasetDetails(pio, referencedTraits, program); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java index 174150992..977306c7f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java @@ -9,6 +9,8 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.utilities.Utilities; @@ -107,4 +109,28 @@ public Map> mapPendingLocationByOUI return locationByOUId; } + + // TODO: used by expunit workflow + private void fetchOrCreateLocationPIO(ImportContext importContext, ExpUnitContext expUnitContext) { + PendingImportObject pio; + String envLocationName = pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } + + // TODO: used by create workflow + private void fetchOrCreateLocationPIO(ImportContext importContext) { + PendingImportObject pio; + String envLocationName = importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); + pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); + this.locationByName.put(envLocationName, pio); + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java new file mode 100644 index 000000000..1e2c121cd --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -0,0 +1,127 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +public class ObservationUnitService { + // TODO: used by expUnit workflow + public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + // TODO: used by create workflow + public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, + PendingData pendingData, + String envSeqValue) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { + PendingImportObject pio; + String key = createObservationUnitKey(importRow); + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { + pio = observationUnitByNameNoScope.get(key); + } else { + String germplasmName = ""; + if (this.existingGermplasmByGID.get(importRow.getGid()) != null) { + germplasmName = this.existingGermplasmByGID.get(importRow.getGid()) + .getBrAPIObject() + .getGermplasmName(); + } + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; + UUID trialID = trialPIO.getId(); + UUID datasetId = null; + if (commit) { + datasetId = UUID.fromString(trialPIO.getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); + } + PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (matchingOU.isEmpty()) { + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); + } + } else { + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } + this.observationUnitByNameNoScope.put(key, pio); + } + return pio; + } + + // TODO: used by both workflows + public String createObservationUnitKey(ExperimentObservation importRow) { + return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId()); + } + + // TODO: used by both workflows + public String createObservationUnitKey(String studyName, String obsUnitName) { + return studyName + obsUnitName; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index b69e805c8..38ae4d9f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -9,19 +9,26 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPISeason; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; +import java.math.BigInteger; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; @Singleton @@ -152,4 +159,99 @@ public Map> mapPendingStudyByOUId( return studyByOUId; } + + // TODO: used by expunit workflow + private PendingImportObject fetchOrCreateStudyPIO( + ImportContext importContext, + ExpUnitContext expUnitContext, + String expSequenceValue, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } + + // TODO: used by create workflow + private PendingImportObject fetchOrCreateStudyPIO( + ImportContext importContext, + String expSequenceValue, + Supplier envNextVal + ) throws UnprocessableEntityException { + PendingImportObject pio; + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { + pio = studyByNameNoScope.get(importRow.getEnv()); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else { + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); + newStudy.setLocationDbId(this.locationByName.get(importRow.getEnvLocation()).getId().toString()); //set as the BI ID to facilitate looking up locations when saving new studies + + // It is assumed that the study has only one season, And that the Years and not + // the dbId's are stored in getSeason() list. + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season + if (commit) { + if(StringUtils.isNotBlank(year)) { + String seasonID = this.yearToSeasonDbId(year, program.getId()); + newStudy.setSeasons(Collections.singletonList(seasonID)); + } + } else { + addYearToStudyAdditionalInfo(program, newStudy, year); + } + + pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); + this.studyByNameNoScope.put(importRow.getEnv(), pio); + } + return pio; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java new file mode 100644 index 000000000..a84b84e30 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java @@ -0,0 +1,112 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import tech.tablesaw.columns.Column; + +import java.util.List; +import java.util.Map; + +public class ValidateService { + // TODO: used by expUnit workflow + public void prepareDataForValidation(ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } + + // TODO: used by create workflow + public void prepareDataForValidation(ImportContext importContext, + List> phenotypeCols) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); + List> observations = mappedImportRow.getObservations(); + String observationHash; + if (hasAllReferenceUnitIds) { + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } + + mappedBrAPIImport.put(rowNum, mappedImportRow); + } + } +} From 63091c3ccff9c83972f903e68fb418b1d5f7aa64 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 12:48:22 -0400 Subject: [PATCH 18/61] add ValidateService#validateFields --- .../experiment/create/model/PendingData.java | 2 + .../experiment/services/ValidateService.java | 66 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index c3f769a71..340a39fba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -6,6 +6,7 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; @@ -25,4 +26,5 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java index a84b84e30..8004a5548 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java @@ -1,16 +1,25 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; import tech.tablesaw.columns.Column; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class ValidateService { // TODO: used by expUnit workflow @@ -109,4 +118,61 @@ public void prepareDataForValidation(ImportContext importContext, mappedBrAPIImport.put(rowNum, mappedImportRow); } } + + // TODO: used by expUnit workflow + public void validateFields(ImportContext importContext, + PendingData pendingData, + ExpUnitContext expUnitContext, + List referencedTraits, Program program, + List> phenotypeCols) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + } + } + + // TODO: used by create workflow + public void validateFields(ImportContext importContext, + PendingData pendingData, + List referencedTraits, Program program, + List> phenotypeCols) { + //fetching any existing observations for any OUs in the import + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } + Set uniqueStudyAndObsUnit = new HashSet<>(); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } + } + } } From 8a89c533fdbba964694cdae4b7120d6eefc56ee1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 3 May 2024 15:36:39 -0400 Subject: [PATCH 19/61] add observation and statistics service --- .../services/ObservationService.java | 143 ++++++++++++++++++ .../services/ObservationUnitService.java | 19 +++ .../services/StatisticsService.java | 92 +++++++++++ .../experiment/services/ValidateService.java | 137 +++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java new file mode 100644 index 000000000..287e4e595 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java @@ -0,0 +1,143 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Trait; +import org.breedinginsight.model.User; +import tech.tablesaw.columns.Column; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +public class ObservationService { + // TODO: used with expUnit workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + validateObservation(importHash); + } + } + + // TODO: used with create workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + + + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash(importRow, phenoCol.name()); + + validateObservation(importHash); + } + + } + + // TODO: used by both workflows + private void validateObservation(String importHash) { + + + String importObsValue = phenoCol.getString(rowNum); + + + // error if import observation data already exists and user has not selected to overwrite + if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if (timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java index 1e2c121cd..b0719223b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -1,9 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.services; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -16,6 +18,7 @@ import org.breedinginsight.utilities.Utilities; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -124,4 +127,20 @@ public String createObservationUnitKey(ExperimentObservation importRow) { public String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + + // TODO: used by create workflow + public void validateObservationUnits(ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow) { + validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + + String key = createObservationUnitKey(importRow); + PendingImportObject ouPIO = observationUnitByNameNoScope.get(key); + if(ouPIO.getState() == ImportObjectState.NEW && StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "Could not find observation unit by ObsUnitDBID", validationErrors, rowNum); + } + + validateGeoCoordinates(validationErrors, rowNum, importRow); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java new file mode 100644 index 000000000..26e6d7f17 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java @@ -0,0 +1,92 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.services; + +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.services.exceptions.ValidatorException; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class StatisticsService { + // TODO: used by both workflows + public Map generateStatisticsMap(List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's + + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, createObservationUnitKey(importRow)); + addIfNotNull(gidCounter, importRow.getGid()); + } + + int numNewObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numExistingObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameCounter.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDCounter.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(gidCounter.size()) + .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); + ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numExistingObservations) + .build(); + ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numMutatedObservations) + .build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats, + "Existing_Observations", existingObservationStats, + "Mutated_Observations", mutatedObservationStats + ); + } + + // TODO: used by both workflows + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + // TODO + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java index 8004a5548..bb0db1662 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java @@ -7,6 +7,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -14,6 +15,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import java.util.HashSet; @@ -175,4 +177,139 @@ public void validateFields(ImportContext importContext, } } } + + // TODO: used by create workflow + private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { + String testOrCheck = importRow.getTestOrCheck(); + if ( ! ( testOrCheck==null || testOrCheck.isBlank() + || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) + || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) + ){ + addRowError(ExperimentObservation.Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; + } + } + + // TODO: used by create workflow + private void validateConditionallyRequired(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, Program program, boolean commit) { + ImportObjectState expState = this.trialByNameNoScope.get(importRow.getExpTitle()) + .getState(); + ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); + + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } + + if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { + validateRequiredCell(importRow.getGid(), ExperimentObservation.Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpTitle(), ExperimentObservation.Columns.EXP_TITLE,errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpUnit(), ExperimentObservation.Columns.EXP_UNIT, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpType(), ExperimentObservation.Columns.EXP_TYPE, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getEnv(), ExperimentObservation.Columns.ENV, errorMessage, validationErrors, rowNum); + if(validateRequiredCell(importRow.getEnvLocation(), ExperimentObservation.Columns.ENV_LOCATION, errorMessage, validationErrors, rowNum)) { + if(!Utilities.removeProgramKeyAndUnknownAdditionalData(this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getLocationName(), program.getKey()).equals(importRow.getEnvLocation())) { + addRowError(ExperimentObservation.Columns.ENV_LOCATION, ENV_LOCATION_MISMATCH, validationErrors, rowNum); + } + } + if(validateRequiredCell(importRow.getEnvYear(), ExperimentObservation.Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { + String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); + String rowYear = importRow.getEnvYear(); + if(commit) { + rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); + } + if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { + addRowError(ExperimentObservation.Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); + } + } + validateRequiredCell(importRow.getExpUnitId(), ExperimentObservation.Columns.EXP_UNIT_ID, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpReplicateNo(), ExperimentObservation.Columns.REP_NUM, errorMessage, validationErrors, rowNum); + validateRequiredCell(importRow.getExpBlockNo(), ExperimentObservation.Columns.BLOCK_NUM, errorMessage, validationErrors, rowNum); + + if(StringUtils.isNotBlank(importRow.getObsUnitID())) { + addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); + } + } else { + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + ExperimentObservation.Columns.OBS_UNIT_ID, + MISSING_OBS_UNIT_ID_ERROR, + validationErrors, + rowNum + ); + } + } + + // TODO: used by create workflow + public void validateGeoCoordinates(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow) { + + String lat = importRow.getLatitude(); + String lon = importRow.getLongitude(); + String elevation = importRow.getElevation(); + + // If any of Lat, Long, or Elevation are provided, Lat and Long must both be provided. + if (StringUtils.isNotBlank(lat) || StringUtils.isNotBlank(lon) || StringUtils.isNotBlank(elevation)) { + if (StringUtils.isBlank(lat)) { + addRowError(ExperimentObservation.Columns.LAT, "Latitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + if (StringUtils.isBlank(lon)) { + addRowError(ExperimentObservation.Columns.LONG, "Longitude must be provided for complete coordinate specification", validationErrors, rowNum); + } + } + + // Validate coordinate values + boolean latBadValue = false; + boolean lonBadValue = false; + boolean elevationBadValue = false; + double latDouble; + double lonDouble; + double elevationDouble; + + // Only check latitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lat)) { + try { + latDouble = Double.parseDouble(lat); + if (latDouble < -90 || latDouble > 90) { + latBadValue = true; + } + } catch (NumberFormatException e) { + latBadValue = true; + } + } + + // Only check longitude format if not blank since already had previous error + if (StringUtils.isNotBlank(lon)) { + try { + lonDouble = Double.parseDouble(lon); + if (lonDouble < -180 || lonDouble > 180) { + lonBadValue = true; + } + } catch (NumberFormatException e) { + lonBadValue = true; + } + } + + if (StringUtils.isNotBlank(elevation)) { + try { + elevationDouble = Double.parseDouble(elevation); + } catch (NumberFormatException e) { + elevationBadValue = true; + } + } + + if (latBadValue) { + addRowError(ExperimentObservation.Columns.LAT, "Invalid Lat value (expected range -90 to 90)", validationErrors, rowNum); + } + + if (lonBadValue) { + addRowError(ExperimentObservation.Columns.LONG, "Invalid Long value (expected range -180 to 180)", validationErrors, rowNum); + } + + if (elevationBadValue) { + addRowError(ExperimentObservation.Columns.LONG, "Invalid Elevation value (numerals expected)", validationErrors, rowNum); + } + + } } From 42b1482f9266ab220e7016f30851db1756dd1ea7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 14:15:33 -0400 Subject: [PATCH 20/61] update Middleware#link to handle nested middleware --- .../processors/experiment/middleware/Middleware.java | 10 +++++++--- .../processors/experiment/services/DatasetService.java | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index d2d7443b8..6d097722f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -13,9 +13,9 @@ public abstract class Middleware { public static Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { - nextInChain.prior = head; - head.next = nextInChain; - head = nextInChain; + nextInChain.prior = head.getLastLink(); + head.getLastLink().next = nextInChain; + head = nextInChain.getLastLink(); } return first; } @@ -49,4 +49,8 @@ protected boolean compensatePrior(T context, MiddlewareError error) { } return prior.compensate(context, error); } + + private Middleware getLastLink() { + return this.next == null ? this : this.next.getLastLink(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index db5fcdedb..83bf01381 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -185,4 +185,6 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, } addObsVarsToDatasetDetails(pio, referencedTraits, program); } + + } From 7ca5817dad0fc2b26dea8a0e2bd421510bb1b102 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 15:41:18 -0400 Subject: [PATCH 21/61] add method to create new brapi trials for import --- .../postBrAPIData/CreateBrAPITrials.java | 4 +++ .../services/ObservationService.java | 26 +++++++++++++++++++ .../services/ObservationUnitService.java | 21 +++++++++++++++ .../experiment/services/TrialService.java | 15 +++++++++++ 4 files changed, 66 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java new file mode 100644 index 000000000..b84b0bd3e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java @@ -0,0 +1,4 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.postBrAPIData; + +public class CreateBrAPITrials { +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java index 287e4e595..d57eca250 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java @@ -10,9 +10,11 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; import tech.tablesaw.columns.Column; @@ -140,4 +142,28 @@ private void validateObservation(String importHash) { } } + + // TODO: used by both workflows + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : this.observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java index b0719223b..510ef75b1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -18,6 +18,7 @@ import org.breedinginsight.utilities.Utilities; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -143,4 +144,24 @@ public void validateObservationUnits(ValidationErrors validationErrors, validateGeoCoordinates(validationErrors, rowNum, importRow); } + + // TODO: used by both workflows + private void updateObsUnitDependencyValues(String programKey) { + + // update study DbIds + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); + + // update germplasm DbIds + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 83b36f9d7..325347d0c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -15,6 +15,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; @@ -260,4 +261,18 @@ public PendingImportObject fetchOrCreateTrialPIO( return trialPio; } + + // TODO: used by both workflows + List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { + List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); + List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); + // set the DbId to the for each newly created trial + for (BrAPITrial createdTrial : createdTrials) { + String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + this.trialByNameNoScope.get(createdTrialName) + .getBrAPIObject() + .setTrialDbId(createdTrial.getTrialDbId()); + } + return createdTrials; + } } From afc5e918942dd6c38125e7ff117cb10cb89af1cb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 15:47:44 -0400 Subject: [PATCH 22/61] add method to update pending trials --- .../experiment/services/TrialService.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 325347d0c..749599179 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -275,4 +275,24 @@ List commitNewPendingTrialsToBrAPIStore(ImportContext context, Pendi } return createdTrials; } + + List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedTrials = new ArrayList<>(); + Map mutatedTrialsById = ProcessorData + .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); + for (Map.Entry entry : mutatedTrialsById.entrySet()) { + String id = entry.getKey(); + BrAPITrial trial = entry.getValue(); + try { + updatedTrials.add(brapiTrialDAO.updateBrAPITrial(id, trial, program.getId())); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + return updatedTrials; + } } From c926da4ec16c8c7e76a96be7abff9f4bdabc66a3 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 16:03:16 -0400 Subject: [PATCH 23/61] add methods for creating and updating pending datasets --- .../experiment/services/DatasetService.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index 83bf01381..372afe408 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -5,12 +5,14 @@ import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -20,6 +22,7 @@ import org.breedinginsight.utilities.Utilities; import java.util.*; +import java.util.stream.Collectors; public class DatasetService { // TODO: used by expunit worflow @@ -186,5 +189,50 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, addObsVarsToDatasetDetails(pio, referencedTraits, program); } + // TODO: used by both workflows + List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { + BrAPIListNewRequest request = new BrAPIListNewRequest(); + request.setListName(details.getListName()); + request.setListType(details.getListType()); + request.setExternalReferences(details.getExternalReferences()); + request.setAdditionalInfo(details.getAdditionalInfo()); + request.data(details.getData()); + return request; + }).collect(Collectors.toList()); + List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); + for (BrAPIListSummary summary : createdDatasets) { + obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); + } + return createdDatasets; + } + // TODO: used by both workflows + List commitUpdatedPendingDatasetToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedDatasets = new ArrayList<>(); + Map datasetNewDataById = ProcessorData + .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); + for (Map.Entry entry : datasetNewDataById.entrySet()) { + String id = entry.getKey(); + BrAPIListDetails dataset = entry.getValue(); + try { + List existingObsVarIds = brAPIListDAO.getListById(id, program.getId()).getResult().getData(); + List newObsVarIds = dataset + .getData() + .stream() + .filter(obsVarId -> !existingObsVarIds.contains(obsVarId)).collect(Collectors.toList()); + List obsVarIds = new ArrayList<>(existingObsVarIds); + obsVarIds.addAll(newObsVarIds); + dataset.setData(obsVarIds); + updatedDatasets.add(brAPIListDAO.updateBrAPIList(id, dataset, program.getId())); + } catch (ApiException e) { + log.error("Error updating dataset observation variables: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating dataset observation variables: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + return updatedDatasets; + } } From 6ae624e76255ca75569d3e5c17b738f32ab46ea9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 16:54:23 -0400 Subject: [PATCH 24/61] add method to update brapi study dependencies --- .../experiment/services/DatasetService.java | 4 +- .../experiment/services/LocationService.java | 36 ++++++++++++++++++ .../experiment/services/StudyService.java | 38 +++++++++++++++++++ .../experiment/services/TrialService.java | 4 +- 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java index 372afe408..2a7cd2897 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java @@ -190,7 +190,7 @@ public void fetchOrCreateDatasetPIO(ImportContext importContext, } // TODO: used by both workflows - List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + public List commitNewPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { List newDatasetRequests = ProcessorData.getNewObjects(obsVarDatasetByName).stream().map(details -> { BrAPIListNewRequest request = new BrAPIListNewRequest(); request.setListName(details.getListName()); @@ -208,7 +208,7 @@ List commitNewPendingDatasetsToBrAPIStore(ImportContext import } // TODO: used by both workflows - List commitUpdatedPendingDatasetToBrAPIStore(ImportContext importContext, PendingData pendingData) { + public List commitUpdatedPendingDatasetsToBrAPIStore(ImportContext importContext, PendingData pendingData) { List updatedDatasets = new ArrayList<>(); Map datasetNewDataById = ProcessorData .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java index 977306c7f..29e2c851d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java @@ -4,12 +4,19 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.request.BrAPIListNewRequest; +import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -133,4 +140,33 @@ private void fetchOrCreateLocationPIO(ImportContext importContext) { this.locationByName.put(envLocationName, pio); } } + + // TODO: used by both workflows + public List commitNewPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + AuthenticatedUser actingUser = new AuthenticatedUser(upload.getUpdatedByUser().getName(), new ArrayList<>(), upload.getUpdatedByUser().getId(), new ArrayList<>()); + + List newLocations = ProcessorData.getNewObjects(this.locationByName) + .stream() + .map(location -> ProgramLocationRequest.builder() + .name(location.getName()) + .build()) + .collect(Collectors.toList()); + + List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); + // set the DbId to the for each newly created location + for (ProgramLocation createdLocation : createdLocations) { + String createdLocationName = createdLocation.getName(); + this.locationByName.get(createdLocationName) + .getBrAPIObject() + .setLocationDbId(createdLocation.getLocationDbId()); + } + return createdLocations; + } + + // TODO: used by both workflows + public List commitUpdatedPendingLocationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedLocations = new ArrayList<>(); + + return updatedLocations; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index 38ae4d9f9..1a5c61b05 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -13,6 +13,7 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -21,6 +22,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; @@ -254,4 +256,40 @@ private PendingImportObject fetchOrCreateStudyPIO( } return pio; } + + public void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { + // update location DbIds in studies for all distinct locations + mappedBrAPIImport.values() + .stream() + .map(PendingImport::getLocation) + .forEach(this::updateStudyLocationDbId); + + // update trial DbIds in studies for all distinct trials + this.trialByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(trial -> this.updateTrialDbId(trial, programKey)); + } + + private void updateStudyLocationDbId(PendingImportObject location) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> location.getId().toString() + .equals(study.getBrAPIObject() + .getLocationDbId())) + .forEach(study -> study.getBrAPIObject() + .setLocationDbId(location.getBrAPIObject().getLocationDbId())); + } + + private void updateTrialDbId(BrAPITrial trial, String programKey) { + this.studyByNameNoScope.values() + .stream() + .filter(study -> study.getBrAPIObject() + .getTrialName() + .equals(Utilities.removeProgramKey(trial.getTrialName(), programKey))) + .forEach(study -> study.getBrAPIObject() + .setTrialDbId(trial.getTrialDbId())); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java index 749599179..57ec2d52b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java @@ -263,7 +263,7 @@ public PendingImportObject fetchOrCreateTrialPIO( } // TODO: used by both workflows - List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { + public List commitNewPendingTrialsToBrAPIStore(ImportContext context, PendingData pendingData) { List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial @@ -276,7 +276,7 @@ List commitNewPendingTrialsToBrAPIStore(ImportContext context, Pendi return createdTrials; } - List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + public List commitUpdatedPendingTrialsToBrAPIStore(ImportContext importContext, PendingData pendingData) { List updatedTrials = new ArrayList<>(); Map mutatedTrialsById = ProcessorData .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); From 314d0b129d6f563b2b2bf34cbebe51f8fb7bf0a5 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 6 May 2024 17:07:01 -0400 Subject: [PATCH 25/61] Added db migration to create workflows --- .../V1.22.0__add_experiment_workflows.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql new file mode 100644 index 000000000..a78947c38 --- /dev/null +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -0,0 +1,41 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE importer_mapping_workflow +( + like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, + mapping_id UUID NOT NULL, + name TEXT +); + +ALTER TABLE importer_mapping_workflow + ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); + +DO +$$ +DECLARE + exp_mapping_id UUID; +BEGIN + exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); + +INSERT INTO public.importer_mapping_workflow (mapping_id, name) +VALUES + (exp_mapping_id, 'Create new experiment'), + (exp_mapping_id, 'Append experimental dataset'), + (exp_mapping_id, 'Create new experimental environment'); +END +$$; \ No newline at end of file From 17ec46143393cd65123b721cbb3e6b2bf971cc90 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 17:15:45 -0400 Subject: [PATCH 26/61] add methods for committing studies --- .../experiment/services/StudyService.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index 1a5c61b05..6736a7c28 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -2,6 +2,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.server.exceptions.InternalServerException; import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -18,8 +19,10 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; @@ -257,7 +260,7 @@ private PendingImportObject fetchOrCreateStudyPIO( return pio; } - public void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { + private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations mappedBrAPIImport.values() .stream() @@ -292,4 +295,20 @@ private void updateTrialDbId(BrAPITrial trial, String programKey) { .forEach(study -> study.getBrAPIObject() .setTrialDbId(trial.getTrialDbId())); } + + // TODO: used by both workflows + public List commitNewPendingStudiessToBrAPIStore(ImportContext context, PendingData pendingData) { + List newStudies = ProcessorData.getNewObjects(this.studyByNameNoScope); + updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); + List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + + return createdStudies; + } + + // TODO: used by both workflows + public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedStudies = new ArrayList<>(); + + return updatedStudies; + } } From 968e9baf363301d5e0b55561e8cb694b9d783f1b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 17:33:15 -0400 Subject: [PATCH 27/61] add methods for committing obs units --- .../services/ObservationUnitService.java | 80 ++++++++++++++++++- .../experiment/services/StudyService.java | 10 ++- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java index 510ef75b1..8dbcbc2d2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java @@ -4,23 +4,23 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; public class ObservationUnitService { @@ -164,4 +164,76 @@ private void updateObsUnitDependencyValues(String programKey) { .map(PendingImportObject::getBrAPIObject) .forEach(this::updateGermplasmDbId); } + + // TODO: used by both workflows + public List commitNewPendingObservationUnitsToBrAPIStore(ImportContext context, PendingData pendingData) { + List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + updateObsUnitDependencyValues(program.getKey()); + List createdObservationUnits = brAPIObservationUnitDAO.createBrAPIObservationUnits(newObservationUnits, program.getId(), upload); + + // set the DbId to the for each newly created Observation Unit + for (BrAPIObservationUnit createdObservationUnit : createdObservationUnits) { + // retrieve the BrAPI ObservationUnit from this.observationUnitByNameNoScope + String createdObservationUnit_StripedStudyName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getStudyName(), program.getKey()); + String createdObservationUnit_StripedObsUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData(createdObservationUnit.getObservationUnitName(), program.getKey()); + String createdObsUnit_key = createObservationUnitKey(createdObservationUnit_StripedStudyName, createdObservationUnit_StripedObsUnitName); + this.observationUnitByNameNoScope.get(createdObsUnit_key) + .getBrAPIObject() + .setObservationUnitDbId(createdObservationUnit.getObservationUnitDbId()); + } + + return createdObservationUnits; + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationUnitToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedUnits = new ArrayList<>(); + + return updatedUnits; + } + + private void updateObsUnitDependencyValues(String programKey) { + + // update study DbIds + this.studyByNameNoScope.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(study -> updateStudyDbId(study, programKey)); + + // update germplasm DbIds + this.existingGermplasmByGID.values() + .stream() + .filter(Objects::nonNull) + .distinct() + .map(PendingImportObject::getBrAPIObject) + .forEach(this::updateGermplasmDbId); + } + + private void updateStudyDbId(BrAPIStudy study, String programKey) { + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> obsUnit.getBrAPIObject() + .getStudyName() + .equals(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey))) + .forEach(obsUnit -> { + obsUnit.getBrAPIObject() + .setStudyDbId(study.getStudyDbId()); + obsUnit.getBrAPIObject() + .setTrialDbId(study.getTrialDbId()); + }); + } + + private void updateGermplasmDbId(BrAPIGermplasm germplasm) { + this.observationUnitByNameNoScope.values() + .stream() + .filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) + .forEach(obsUnit -> obsUnit.getBrAPIObject() + .setGermplasmDbId(germplasm.getGermplasmDbId())); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java index 6736a7c28..5ff7a2729 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java @@ -302,13 +302,21 @@ public List commitNewPendingStudiessToBrAPIStore(ImportContext conte updateStudyDependencyValues(mappedBrAPIImport, program.getKey()); List createdStudies = brAPIStudyDAO.createBrAPIStudies(newStudies, program.getId(), upload); + // set the DbId to the for each newly created study + for (BrAPIStudy createdStudy : createdStudies) { + String createdStudy_name_no_key = Utilities.removeProgramKeyAndUnknownAdditionalData(createdStudy.getStudyName(), program.getKey()); + this.studyByNameNoScope.get(createdStudy_name_no_key) + .getBrAPIObject() + .setStudyDbId(createdStudy.getStudyDbId()); + } + return createdStudies; } // TODO: used by both workflows public List commitUpdatedPendingStudiesToBrAPIStore(ImportContext importContext, PendingData pendingData) { List updatedStudies = new ArrayList<>(); - + return updatedStudies; } } From 70d268cd18101bdc7a681f893ffcd562df1c0afb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 6 May 2024 17:57:00 -0400 Subject: [PATCH 28/61] add methods for committing pending observations --- .../services/ObservationService.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java index d57eca250..dab65dd5c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java @@ -2,26 +2,35 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; public class ObservationService { // TODO: used with expUnit workflow @@ -166,4 +175,60 @@ private void updateObservationDependencyValues(Program program) { } } } + + // TODO: used by both workflows + public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + updateObservationDependencyValues(program); + return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedObservations = new ArrayList<>(); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String id = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + updatedObservations.add(updatedObs); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + return updatedObservations; + } } From 8fc2f31a33698340a15d42dd8c80da7561c76684 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 11:33:56 -0400 Subject: [PATCH 29/61] Added bean mapping for factory producing appropriate workflow --- .../migration/V1.22.0__add_experiment_workflows.sql | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql index a78947c38..d5c4b9556 100644 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -19,7 +19,8 @@ CREATE TABLE importer_mapping_workflow ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, mapping_id UUID NOT NULL, - name TEXT + name TEXT NOT NULL, + bean TEXT NOT NULL ); ALTER TABLE importer_mapping_workflow @@ -32,10 +33,10 @@ DECLARE BEGIN exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); -INSERT INTO public.importer_mapping_workflow (mapping_id, name) +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean) VALUES - (exp_mapping_id, 'Create new experiment'), - (exp_mapping_id, 'Append experimental dataset'), - (exp_mapping_id, 'Create new experimental environment'); + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow'), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow'), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow'); END $$; \ No newline at end of file From 4b61708eaa9ece75443f6608b20bdf389b165c93 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 11:36:12 -0400 Subject: [PATCH 30/61] Added endpoint for retrieving workflows for given mapping id --- .../controllers/ImportController.java | 18 +++++++ .../daos/ImportMappingWorkflowDAO.java | 46 ++++++++++++++++ .../model/workflow/ImportMappingWorkflow.java | 52 +++++++++++++++++++ .../importer/services/FileImportService.java | 14 ++++- 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e0a61117a..5c9ac0ca1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -208,4 +209,21 @@ public HttpResponse>> getSystemMappings(@Nu Response> response = new Response(metadata, new DataResponse<>(result)); return HttpResponse.ok(response); } + + @Get("/import/mappings/{mappingId}/workflows") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @Secured(SecurityRule.IS_ANONYMOUS) + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + + List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + + List metadataStatus = new ArrayList<>(); + metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); + Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); + Metadata metadata = new Metadata(pagination, metadataStatus); + + Response> response = new Response(metadata, new DataResponse<>(workflows)); + return HttpResponse.ok(response); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java new file mode 100644 index 000000000..e6a0a7104 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -0,0 +1,46 @@ +package org.breedinginsight.brapps.importer.daos; + +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; +import org.jooq.Configuration; +import org.jooq.DSLContext; +import org.jooq.Record; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.breedinginsight.dao.db.Tables.*; + +public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { + + private DSLContext dsl; + + @Inject + public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + /** + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. + * + * @param mappingId The UUID of the mapping to retrieve the workflows for. + * @return A list of ImportMappingWorkflow objects. + */ + public List getWorkflowsByImportMappingId(UUID mappingId) { + + List records = dsl.select() + .from(IMPORTER_MAPPING_WORKFLOW) + .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .fetch(); + + List workflows = new ArrayList<>(); + for (Record record: records) { + workflows.add(ImportMappingWorkflow.parseSQLRecord(record)); + } + return workflows; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java new file mode 100644 index 000000000..51a70bb5a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -0,0 +1,52 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; + +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; +import org.jooq.Record; + +import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@NoArgsConstructor +@SuperBuilder() +public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { + this.setId(importMappingWorkflowEntity.getId()); + this.setName(importMappingWorkflowEntity.getName()); + } + + public static ImportMappingWorkflow parseSQLRecord(Record record) { + + return ImportMappingWorkflow.builder() + .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) + .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .build(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 05c48601d..950465459 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -32,6 +33,7 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -40,8 +42,10 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; +import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -80,12 +84,13 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -97,6 +102,7 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; + this.importMappingWorkflowDAO = importMappingWorkflowDAO; } public List getAllImportTypeConfigs() { @@ -559,4 +565,10 @@ public List getSystemMappingByName(String name) { List importMappings = importMappingDAO.getSystemMappingByName(name); return importMappings; } + + public List getWorkflowsForSystemMapping(UUID mappingId) { + return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + } + + } From 585947eaaf460c99ea5c6866ec8052a3d4e0424f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 May 2024 12:12:20 -0400 Subject: [PATCH 31/61] fix up middleware for refID validation --- .../experiment/ExperimentUtilities.java | 34 +++++++++++++++++++ .../AppendOverwritePhenotypesWorkflow.java | 9 ++--- .../ExpUnitContextService.java | 6 ++-- .../middleware/GetExistingBrAPIData.java | 5 ++- .../middleware/ValidateAllRowsHaveIDs.java | 33 ++++-------------- .../CreateBrAPITrials.java | 2 +- ...ts.java => ExpImportProcessConstants.java} | 6 ++-- .../model/ExpUnitMiddlewareContext.java | 5 ++- .../experiment/model/ProcessedData.java | 5 ++- 9 files changed, 57 insertions(+), 48 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{postBrAPIData => commit}/CreateBrAPITrials.java (65%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/{ExpImportProcessErrorConstants.java => ExpImportProcessConstants.java} (78%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 25b87a0e8..d22f9ae2f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -1,15 +1,21 @@ package org.breedinginsight.brapps.importer.services.processors.experiment; import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class ExperimentUtilities { @@ -59,4 +65,32 @@ public void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Stri additionalInfo.addProperty(BrAPIAdditionalInfoFields.ENV_YEAR, year); } } + + public static Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { + Set referenceOUIds = new HashSet<>(); + boolean hasNoReferenceUnitIds = true; + boolean hasAllReferenceUnitIds = true; + for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) { + + // can't proceed if the import has a mix of ObsUnitId for some but not all rows + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR); + } + return referenceOUIds; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index fe2e1ae3b..d7de40d00 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -17,18 +17,19 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; + Provider validateAllRowsHaveIDsProvider; Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider getExistingBrAPIDataProvider) { + public AppendOverwritePhenotypesWorkflow(Provider validateAllRowsHaveIDsProvider, + Provider getExistingBrAPIDataProvider) { this.middleware.link( - new ValidateAllRowsHaveIDs(), + validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get() ); } @Override public ProcessedData process(ImportContext context) { - ExpUnitMiddlewareContext workflowContext = new ExpUnitMiddlewareContext(); - workflowContext.setImportContext(context); + ExpUnitMiddlewareContext workflowContext = ExpUnitMiddlewareContext.builder().importContext(context).build(); this.middleware.process(workflowContext); // TODO: implement diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 85a415717..4f2bd4411 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite; import io.micronaut.context.annotation.Property; -import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -12,9 +11,8 @@ import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -79,7 +77,7 @@ public Map> fetchReferenceObse missingIds.removeAll(fetchedIds); // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); } return pendingUnitById; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java index e87583d6b..9d85306d4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/GetExistingBrAPIData.java @@ -10,8 +10,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessErrorConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.utilities.Utilities; @@ -96,7 +95,7 @@ private Map> fetchReferenceObs // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store this.compensate(context, new MiddlewareError(() -> { - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessErrorConstants.COMMA_DELIMITER, missingIds)); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); })); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java index 45c728453..4a90c4bec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ValidateAllRowsHaveIDs.java @@ -1,6 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; +import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -8,13 +10,12 @@ import java.util.HashSet; import java.util.Set; +@Slf4j public class ValidateAllRowsHaveIDs extends ExpUnitMiddleware { - private boolean hasAllReferenceUnitIds = true; - private boolean hasNoReferenceUnitIds = true; - @Override public boolean process(ExpUnitMiddlewareContext context) { - context.setReferenceOUIds(collateReferenceOUIds(context)); + + context.getExpUnitContext().setReferenceOUIds(ExperimentUtilities.collateReferenceOUIds(context)); return processNext(context); } @@ -27,27 +28,5 @@ public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError erro return compensatePrior(context, error); } - private Set collateReferenceOUIds(ExpUnitMiddlewareContext context) { - Set referenceOUIds = new HashSet<>(); - for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) { - ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - - // Check if ObsUnitID is blank - if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { - hasAllReferenceUnitIds = false; - } else if (referenceOUIds.contains(importRow.getObsUnitID())) { - - // Throw exception if ObsUnitID is repeated - this.compensate(context, new MiddlewareError(()->{ - throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); - })); - - } else { - // Add ObsUnitID to referenceOUIds - referenceOUIds.add(importRow.getObsUnitID()); - hasNoReferenceUnitIds = false; - } - } - return referenceOUIds; - } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java similarity index 65% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java index b84b0bd3e..d6ea6360b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/postBrAPIData/CreateBrAPITrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/commit/CreateBrAPITrials.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.postBrAPIData; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit; public class CreateBrAPITrials { } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java similarity index 78% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 8509d74f6..4da4414a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessErrorConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -2,16 +2,16 @@ import com.fasterxml.jackson.annotation.JsonValue; -public class ExpImportProcessErrorConstants { +public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; - public enum ExpImportProcessErrMessage { + public enum ErrMessage { MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); private String value; - ExpImportProcessErrMessage(String value) { + ErrMessage(String value) { this.value = value; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java index 740707142..1e81ff833 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpUnitMiddlewareContext.java @@ -4,10 +4,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -@Getter -@Setter +@Data +@Builder public class ExpUnitMiddlewareContext { - private ImportContext importContext; private ExpUnitContext expUnitContext; private PendingData pendingData; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java index 551af741b..49fe30b12 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ProcessedData.java @@ -5,11 +5,10 @@ import java.util.Map; -@Getter -@Setter +@Data @Builder @ToString @NoArgsConstructor public class ProcessedData { - Map statistics; + private Map statistics; } From 6d56b6ef8f58a0b8639da9613213ea61f71bc81e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 May 2024 14:04:34 -0400 Subject: [PATCH 32/61] add error handler middleware --- .../AppendOverwritePhenotypesWorkflow.java | 10 ++++++---- .../appendoverwrite/middleware/HandleErr.java | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index d7de40d00..125b52d14 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -3,6 +3,7 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.HandleErr; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -17,15 +18,16 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; + Provider handleErrProvider; Provider validateAllRowsHaveIDsProvider; Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider validateAllRowsHaveIDsProvider, + public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, + Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { - this.middleware.link( + this.middleware.link(handleErrProvider.get(), validateAllRowsHaveIDsProvider.get(), - getExistingBrAPIDataProvider.get() - ); + getExistingBrAPIDataProvider.get()); } @Override public ProcessedData process(ImportContext context) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java new file mode 100644 index 000000000..267643d63 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java @@ -0,0 +1,20 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; + +@Slf4j +public class HandleErr extends ExpUnitMiddleware { + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return processNext(context); + } + + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // TODO: handle any error here + + return true; + } +} From c1fe8fa55d4b4170cf52c3b82fef2f18f5c5c765 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 7 May 2024 17:25:02 -0400 Subject: [PATCH 33/61] Simplify record mapping --- .../importer/daos/ImportMappingWorkflowDAO.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index e6a0a7104..c2ad58c23 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -30,17 +30,10 @@ public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { * @return A list of ImportMappingWorkflow objects. */ public List getWorkflowsByImportMappingId(UUID mappingId) { - - List records = dsl.select() + return dsl.select() .from(IMPORTER_MAPPING_WORKFLOW) .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .fetch(); - - List workflows = new ArrayList<>(); - for (Record record: records) { - workflows.add(ImportMappingWorkflow.parseSQLRecord(record)); - } - return workflows; + .fetch(ImportMappingWorkflow::parseSQLRecord); } } From 69537273b7b124cf3f59f57208781730c93f7d30 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 May 2024 20:14:23 -0400 Subject: [PATCH 34/61] add ExistingObsUnit middleware --- .../ExpUnitContextService.java | 97 +++++++++++++++++-- .../middleware/ExpUnitMiddleware.java | 8 ++ .../read/brapi/ExistingBrAPIData.java | 24 +++++ .../read/brapi/ExistingObservationUnits.java | 72 ++++++++++++++ .../steps/GetExistingProcessingStep.java | 5 +- .../{services => service}/DatasetService.java | 2 +- .../GermplasmService.java | 2 +- .../LocationService.java | 5 +- .../ObservationService.java | 6 +- .../ObservationUnitService.java | 3 +- .../StatisticsService.java | 2 +- .../{services => service}/StudyService.java | 5 +- .../{services => service}/TrialService.java | 6 +- .../ValidateService.java | 5 +- 14 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/DatasetService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/GermplasmService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/LocationService.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/ObservationService.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/ObservationUnitService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/StatisticsService.java (99%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/StudyService.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/TrialService.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{services => service}/ValidateService.java (99%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 4f2bd4411..39338aac7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -6,13 +6,16 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.model.base.ObservationUnit; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -29,12 +32,77 @@ public class ExpUnitContextService { public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; } + public List getReferenceUnits(Set expUnitIds, + Program program) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), + context.getImportContext().getProgram()); + } + public List getReferenceUnits(ExpUnitMiddlewareContext context) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(context.getExpUnitContext().getReferenceOUIds()), + context.getImportContext().getProgram() + ); + } + + public PendingImportObject constructPIOFromExistingUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + return pendingUnitById; + } + + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + context.getImportContext().getProgram().getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + context.getImportContext().getProgram().getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } - public Map> fetchReferenceObservationUnits( - ImportContext importContext, - ExpUnitContext expUnitContext - ) throws ApiException { + return pendingUnitByNameNoScope; + } + public Map> fetchReferenceObservationUnits(ImportContext importContext, + ExpUnitContext expUnitContext) throws ApiException { Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs @@ -87,10 +155,23 @@ public Map> fetchReferenceObse } } - private Map> mapPendingObservationUnitByName( - ExpUnitContext expUnitContext, - ImportContext importContext - ) { + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } + + public Map> mapPendingObservationUnitByName(ImportContext importContext, + ExpUnitContext expUnitContext) { Map> pendingUnitByName = new HashMap<>(); for (Map.Entry> entry : expUnitContext.getPendingObsUnitByOUId().entrySet()) { String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java index 81e4e1a0d..8a6bbcb4f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/ExpUnitMiddleware.java @@ -2,7 +2,15 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; public abstract class ExpUnitMiddleware extends Middleware { + @Override + public boolean compensate(ExpUnitMiddlewareContext context, MiddlewareError error) { + // tag an error if it occurred in this local transaction + error.tag(this.getClass().getName()); + // undo the prior local transaction + return compensatePrior(context, error); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java new file mode 100644 index 000000000..4c0a449e8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java @@ -0,0 +1,24 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Slf4j +public class ExistingBrAPIData extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + Provider existingObservationUnitsProvider; + + @Inject + public ExistingBrAPIData(Provider existingObservationUnitsProvider) { + this.middleware.link(existingObservationUnitsProvider.get()); + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java new file mode 100644 index 000000000..5e48a062b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java @@ -0,0 +1,72 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.ExpUnitContextService; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class ExistingObservationUnits extends ExpUnitMiddleware { + ExpUnitContextService expUnitContextService; + @Inject + public ExistingObservationUnits(ExpUnitContextService expUnitContextService) { + this.expUnitContextService = expUnitContextService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set referenceIds; + List missingIds; + List existingUnits; + List> pendingExistingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + program = context.getImportContext().getProgram(); + try { + // Collect deltabreed-generated exp unit ids listed in the import + referenceIds = context.getExpUnitContext().getReferenceOUIds(); + + // For each id fetch the observation unit from the brapi data store + existingUnits = expUnitContextService.getReferenceUnits(new HashSet<>(referenceIds), program); + if (existingUnits.size() != referenceIds.size()) { + + // Handle case of missing Observation Units in data store + missingIds = expUnitContextService.collectMissingOUIds(new HashSet<>(referenceIds), new ArrayList<>(existingUnits)); + this.compensate(context, new MiddlewareError(() -> { + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); + })); + } + + // Construct pending import objects from the units + pendingExistingUnits = existingUnits.stream().map(expUnitContextService::constructPIOFromExistingUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by Id + pendingUnitById = expUnitContextService.mapPendingUnitById(new ArrayList<>(pendingExistingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = expUnitContextService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingExistingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java index d01324e4f..c4e620620 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/steps/GetExistingProcessingStep.java @@ -3,7 +3,6 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.server.exceptions.InternalServerException; -import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -22,8 +21,8 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.pipeline.ProcessingStep; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.StudyService; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.TrialService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.ProgramLocationService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 2a7cd2897..e8c37983d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index c15e71fd9..386b96125 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index 29e2c851d..624d02842 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -1,13 +1,10 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.request.BrAPIListNewRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.ProgramLocationRequest; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index dab65dd5c..f3ac20000 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -7,11 +7,8 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; @@ -20,7 +17,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 8dbcbc2d2..da879a2ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -15,7 +15,6 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; -import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java index 26e6d7f17..8b9844cdb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StatisticsService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StatisticsService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.lang3.StringUtils; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 5ff7a2729..2356855ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -1,8 +1,6 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.server.exceptions.InternalServerException; import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -15,7 +13,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPISeasonDAO; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 57ec2d52b..37de0d895 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; @@ -9,7 +9,6 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -21,15 +20,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; -import java.math.BigInteger; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Collectors; @Singleton diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java index bb0db1662..dd7bc0af2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ValidateService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ValidateService.java @@ -1,10 +1,9 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.services; +package org.breedinginsight.brapps.importer.services.processors.experiment.service; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -14,13 +13,11 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; public class ValidateService { From 255eb25ab3948ecf5fcd59a9312fff25739b1c1e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 10:23:47 -0400 Subject: [PATCH 35/61] refactor unit service methods and document --- .../ExpUnitContextService.java | 88 +---------- .../read/brapi/ExistingBrAPIData.java | 1 + .../read/brapi/ExistingObservationUnits.java | 32 ++-- .../service/ObservationUnitService.java | 140 ++++++++++++++++++ 4 files changed, 161 insertions(+), 100 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java index 39338aac7..fed70bb51 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/ExpUnitContextService.java @@ -6,15 +6,10 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.base.ObservationUnit; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -35,15 +30,7 @@ public ExpUnitContextService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { public List getReferenceUnits(Set expUnitIds, Program program) throws ApiException { // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), - context.getImportContext().getProgram()); - } - public List getReferenceUnits(ExpUnitMiddlewareContext context) throws ApiException { - // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(context.getExpUnitContext().getReferenceOUIds()), - context.getImportContext().getProgram() - ); + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); } public PendingImportObject constructPIOFromExistingUnit(BrAPIObservationUnit unit) { @@ -90,70 +77,18 @@ public Map> mapPendingUnitByNa for (PendingImportObject pio : pios) { String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( pio.getBrAPIObject().getStudyName(), - context.getImportContext().getProgram().getKey() + program.getKey() ); String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( pio.getBrAPIObject().getObservationUnitName(), - context.getImportContext().getProgram().getKey() + program.getKey() ); pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); } return pendingUnitByNameNoScope; } - public Map> fetchReferenceObservationUnits(ImportContext importContext, - ExpUnitContext expUnitContext) throws ApiException { - Map> pendingUnitById = new HashMap<>(); - try { - // Retrieve reference Observation Units based on IDs - List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( - new ArrayList(expUnitContext.getReferenceOUIds()), - importContext.getProgram() - ); - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - if (referenceObsUnits.size() == expUnitContext.getReferenceOUIds().size()) { - for (BrAPIObservationUnit unit : referenceObsUnits) {// Iterate through reference Observation Units - - // Get external reference for the Observation Unit - Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); - unitXref.ifPresentOrElse( - xref -> { - - // Set pending Observation Unit by its ID - pendingUnitById.put( - xref.getReferenceId(), - new PendingImportObject<>( - ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())) - ); - }, - () -> { - - // but throw an error if no unit ID - throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); - } - ); - } - } else {// Handle case of missing Observation Units in data store - List missingIds = new ArrayList<>(expUnitContext.getReferenceOUIds()); - Set fetchedIds = referenceObsUnits.stream() - .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) - .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - - // throw error reporting any reference IDs with no corresponding stored unit in the brapi data store - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); - } - - return pendingUnitById; - } catch (ApiException e) { - log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new ApiException(e); - } - } public List collectMissingOUIds(Set referenceIds, List existingUnits) { List missingIds = new ArrayList<>(referenceIds); @@ -169,21 +104,4 @@ public List collectMissingOUIds(Set referenceIds, List> mapPendingObservationUnitByName(ImportContext importContext, - ExpUnitContext expUnitContext) { - Map> pendingUnitByName = new HashMap<>(); - for (Map.Entry> entry : expUnitContext.getPendingObsUnitByOUId().entrySet()) { - String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getStudyName(), - importContext.getProgram().getKey() - ); - String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( - entry.getValue().getBrAPIObject().getObservationUnitName(), - importContext.getProgram().getKey() - ); - pendingUnitByName.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), entry.getValue()); - } - return pendingUnitByName; - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java index 4c0a449e8..f3c342438 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java @@ -19,6 +19,7 @@ public ExistingBrAPIData(Provider existingObservationU @Override public boolean process(ExpUnitMiddlewareContext context) { + log.debug("reading required BrAPI data from BrAPI service"); return this.middleware.process(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java index 5e48a062b..6255125c9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java @@ -4,11 +4,11 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.ExpUnitContextService; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.model.Program; import javax.inject.Inject; @@ -17,46 +17,48 @@ @Slf4j public class ExistingObservationUnits extends ExpUnitMiddleware { - ExpUnitContextService expUnitContextService; + ObservationUnitService observationUnitService; + @Inject - public ExistingObservationUnits(ExpUnitContextService expUnitContextService) { - this.expUnitContextService = expUnitContextService; + public ExistingObservationUnits(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; } @Override public boolean process(ExpUnitMiddlewareContext context) { Program program; - Set referenceIds; + Set expUnitIds; List missingIds; - List existingUnits; - List> pendingExistingUnits; + List brapiUnits; + List> pendingUnits; Map> pendingUnitById; Map> pendingUnitByNameNoScope; + log.debug("fetching existing exp units from BrAPI service"); program = context.getImportContext().getProgram(); try { // Collect deltabreed-generated exp unit ids listed in the import - referenceIds = context.getExpUnitContext().getReferenceOUIds(); + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); // For each id fetch the observation unit from the brapi data store - existingUnits = expUnitContextService.getReferenceUnits(new HashSet<>(referenceIds), program); - if (existingUnits.size() != referenceIds.size()) { + brapiUnits = observationUnitService.getReferenceUnits(new HashSet<>(expUnitIds), program); + if (brapiUnits.size() != expUnitIds.size()) { // Handle case of missing Observation Units in data store - missingIds = expUnitContextService.collectMissingOUIds(new HashSet<>(referenceIds), new ArrayList<>(existingUnits)); + missingIds = observationUnitService.collectMissingOUIds(new HashSet<>(expUnitIds), new ArrayList<>(brapiUnits)); this.compensate(context, new MiddlewareError(() -> { throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); })); } // Construct pending import objects from the units - pendingExistingUnits = existingUnits.stream().map(expUnitContextService::constructPIOFromExistingUnit).collect(Collectors.toList()); + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - // Construct a hashmap to look up the pending unit by Id - pendingUnitById = expUnitContextService.mapPendingUnitById(new ArrayList<>(pendingExistingUnits)); + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = expUnitContextService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingExistingUnits), program); + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); // add maps to the context for use in processing import context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index da879a2ba..d7f3708d5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -1,28 +1,168 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import io.micronaut.context.annotation.Property; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; public class ObservationUnitService { + private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { + this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; + } + + /** + * Retrieves the reference Observation Units based on the provided set of experimental unit IDs and the associated Program. + * This function queries the database to fetch Observation Units that match the specified IDs within the given Program. + * + * @param expUnitIds A set of unique identifiers representing the Experimental Units for which reference Observation Units are required. + * These IDs serve as filters to determine the relevant Observation Units. + * @param program The Program to which the Observation Units belong. This parameter ensures that the fetched units are specific to the designated Program. + * @return A List of BrAPIObservationUnit objects that correspond to the reference Observation Units matching the provided expUnitIds within the given Program. + * @throws ApiException If an error occurs during the retrieval process, an ApiException is thrown to handle exceptional scenarios. + */ + public List getReferenceUnits(Set expUnitIds, + Program program) throws ApiException { + // Retrieve reference Observation Units based on IDs + return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); + } + + /** + * Constructs a PendingImportObject of type BrAPIObservationUnit from a given BrAPIObservationUnit object. + * This function is responsible for constructing an import object that represents an observation unit for the Deltabreed system, + * using a provided BrAPIObservationUnit object from a BrAPI source. + * + * @param unit the BrAPIObservationUnit object to be used for constructing the PendingImportObject + * @return a PendingImportObject of type BrAPIObservationUnit representing the imported observation unit + * @throws IllegalStateException if the external reference for the observation unit does not exist + */ + public PendingImportObject constructPIOFromBrapiUnit(BrAPIObservationUnit unit) { + final PendingImportObject[] pio = new PendingImportObject[]{null}; + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + // Get external reference for the Observation Unit + Optional unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource); + unitXref.ifPresentOrElse( + xref -> { + pio[0] = new PendingImportObject(ImportObjectState.EXISTING, unit, UUID.fromString(xref.getReferenceId())); + }, + () -> { + + // but throw an error if no unit ID + throw new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID"); + } + ); + return pio[0]; + } + + /** + * Maps pending observation units by their reference IDs. + * This function takes a list of pending import objects representing BrAPI observation units + * and constructs a map where the key is the external reference ID of the observation unit + * and the value is the pending import object itself. + * + * @param pios List of pending import objects for BrAPI observation units + * @return A map of pending observation units keyed by their external reference ID + */ + public Map> mapPendingUnitById(List> pios) { + Map> pendingUnitById = new HashMap<>(); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + for (PendingImportObject pio : pios) { + + // Get external reference for the Observation Unit + Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); + pendingUnitById.put(xref.get().getReferenceId(),pio); + } + + return pendingUnitById; + } + + /** + * This method takes a list of PendingImportObject objects and a Program object as input + * and maps the PendingImportObject objects by their observation unit name without the program scope. + * + * @param pios A list of PendingImportObject objects to be processed + * @param program The Program object representing the scope to be removed from observation unit names + * @return A Map> mapping observation unit names without the program scope to the corresponding PendingImportObject objects + */ + public Map> mapPendingUnitByNameNoScope(List> pios, + Program program) { + Map> pendingUnitByNameNoScope = new HashMap<>(); + + for (PendingImportObject pio : pios) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pio.getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByNameNoScope.put(ExperimentUtilities.createObservationUnitKey(studyName, observationUnitName), pio); + } + + return pendingUnitByNameNoScope; + } + + /** + * Collects missing Observation Unit IDs from a set of reference IDs and a list of existing Observation Units. + * + * This function takes a Set of reference IDs and a List of existing Observation Units, filters out the Observation Units + * that have external references matching a specific source, and returns a List of missing Observation Unit IDs that are + * present in the reference IDs but not found in the existing Observation Units. + * + * @param referenceIds The Set of reference IDs representing all possible Observation Unit IDs to match against. + * @param existingUnits The List of existing Observation Units to compare against the reference IDs. + * @return A List of Observation Unit IDs that are missing from the existing Observation Units but present in the reference IDs. + */ + public List collectMissingOUIds(Set referenceIds, List existingUnits) { + List missingIds = new ArrayList<>(referenceIds); + + // Construct the DeltaBreed observation unit source for external references + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + Set fetchedIds = existingUnits.stream() + .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) + .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + + return missingIds; + } + // TODO: used by expUnit workflow public PendingImportObject fetchOrCreateObsUnitPIO(ImportContext importContext, PendingData pendingData, From 251d0aed24a19f35786776059de056b6bebef269 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 14:25:06 -0400 Subject: [PATCH 36/61] get trials for required exp units --- ...gBrAPIData.java => RequiredBrAPIData.java} | 6 +- ...its.java => RequiredObservationUnits.java} | 6 +- .../middleware/read/brapi/RequiredTrials.java | 65 +++++++++ .../experiment/service/TrialService.java | 136 ++++++++++++++++++ 4 files changed, 207 insertions(+), 6 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/{ExistingBrAPIData.java => RequiredBrAPIData.java} (81%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/{ExistingObservationUnits.java => RequiredObservationUnits.java} (95%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java similarity index 81% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index f3c342438..539f63ca7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -8,12 +8,12 @@ import javax.inject.Provider; @Slf4j -public class ExistingBrAPIData extends ExpUnitMiddleware { +public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - Provider existingObservationUnitsProvider; + Provider existingObservationUnitsProvider; @Inject - public ExistingBrAPIData(Provider existingObservationUnitsProvider) { + public RequiredBrAPIData(Provider existingObservationUnitsProvider) { this.middleware.link(existingObservationUnitsProvider.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java similarity index 95% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index 6255125c9..d2e6dab21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/ExistingObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -16,11 +16,11 @@ import java.util.stream.Collectors; @Slf4j -public class ExistingObservationUnits extends ExpUnitMiddleware { +public class RequiredObservationUnits extends ExpUnitMiddleware { ObservationUnitService observationUnitService; @Inject - public ExistingObservationUnits(ObservationUnitService observationUnitService) { + public RequiredObservationUnits(ObservationUnitService observationUnitService) { this.observationUnitService = observationUnitService; } @@ -34,7 +34,7 @@ public boolean process(ExpUnitMiddlewareContext context) { Map> pendingUnitById; Map> pendingUnitByNameNoScope; - log.debug("fetching existing exp units from BrAPI service"); + log.debug("fetching required exp units from BrAPI service"); program = context.getImportContext().getProgram(); try { // Collect deltabreed-generated exp unit ids listed in the import diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java new file mode 100644 index 000000000..3761a5a7d --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredTrials extends ExpUnitMiddleware { + TrialService trialService; + + @Inject + public RequiredTrials(TrialService trialService) { + this.trialService = trialService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Map> pendingUnitByNameNoScope; + Set trialDbIds; + List brAPITrials; + List> pendingTrials; + Map> pendingTrialByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no required units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service trials belonging to required units"); + + // Get the dbIds of the trials belonging to the required exp units + trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); + + // Get the trials belonging to required exp units + brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialsBelongingToUnit(dbId, program)).collect(Collectors.toList()); + + // Construct the pending trials from the BrAPI trials + pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending trial by trial name with the program key removed + pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 37de0d895..d768178ba 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -45,6 +45,96 @@ public TrialService(BrAPITrialDAO brAPITrialDAO, this.studyService = studyService; } + /** + * Retrieves the TrialDbId belonging to a pending unit based on the provided BrAPI observation unit and program. + * If the TrialDbId is directly assigned to the unit, it is returned. Otherwise, the TrialDbId + * assigned to the study belonging to the unit is retrieved. + * + * @param pendingUnit The pending import object representing the BrAPI observation unit. + * @param program The program associated with the pending import. + * @return The TrialDbId belonging to the pending unit. + * @throws IllegalStateException if TrialDbId and StudyDbId are not set for an existing ObservationUnit. + * @throws InternalServerException if there is an internal server error while fetching the TrialDbId. + */ + public String getTrialDbIdBelongingToPendingUnit(PendingImportObject pendingUnit, + Program program) throws IllegalStateException, InternalServerException { + String trialDbId = null; + + BrAPIObservationUnit brapiUnit = pendingUnit.getBrAPIObject(); + if (StringUtils.isBlank(brapiUnit.getTrialDbId()) && StringUtils.isBlank(brapiUnit.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + // get the trial directly assigned to the unit + if (StringUtils.isNotBlank(brapiUnit.getTrialDbId())) { + trialDbId = brapiUnit.getTrialDbId(); + } else { + + // or get the trial directly assigned to the study belonging to the unit + String studyDbId = brapiUnit.getStudyDbId(); + try { + trialDbId = getTrialDbIdBelongingToStudy(studyDbId, program); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + return trialDbId; + } + + /** + * Fetches BrAPI trials belonging to the specified trial database IDs and program. + * + * This method retrieves a list of BrAPI trials that are associated with the provided set of trial database IDs + * within the context of the given program. It ensures that all specified trial IDs are found in the result + * to maintain data integrity. + * + * @param trialDbIds A set of trial database IDs indicating the trials to fetch. + * @param program The program object representing the program to which the trials belong. + * @return A list of BrAPITrial objects representing the fetched trials. + * @throws IllegalStateException If not all specified trial database IDs are found in the fetched trials. + * @throws InternalServerException If an error occurs during the API call to fetch the trials. + */ + public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, Program program) { + try { + List trials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(ExperimentUtilities.COMMA_DELIMITER, missingIds)); + } + + return trials; + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + /** + * Fetches a BrAPI trial belonging to a specific unit based on the provided trialDbId and program. + * + * @param trialDbId The unique identifier of the trial to be fetched. + * @param program The program to which the trial belongs. + * @return The BrAPI trial belonging to the specified unit. + * @throws InternalServerException If an internal server error occurs while processing the request. + * @throws IllegalStateException If the trial with the specified trialDbId is not found. + */ + public BrAPITrial fetchBrapiTrialsBelongingToUnit(String trialDbId, Program program) throws InternalServerException { + try { + List trials = brAPITrialDAO.getTrialsByDbIds(Set.of(trialDbId), program); + BrAPITrial trial = trials.get(0); + if (trials.size() == 0) { + throw new IllegalStateException("Trial not found for trialDbId(s): " + trialDbId); + } + + return trial; + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } // TODO: also used in other workflow /** @@ -123,6 +213,30 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr return trialDbIds; } + /** + * Retrieves the trialDbId that belongs to a specific study with the given studyDbId and program. + * + * @param studyDbId The unique identifier for the study in the system. + * @param program The program object associated with the study. + * @return The trialDbId associated with the study. + * @throws ApiException if the study with the provided studyDbId is not found in the BrAPI service. + * @throws IllegalStateException if the trialDbId is not set for the existing study. + */ + private String getTrialDbIdBelongingToStudy(String studyDbId, Program program) throws ApiException { + String trialDbId = null; + List studies = studyService.fetchStudiesByDbId(Set.of(studyDbId), program); + if (studies.size() == 0) { + throw new ApiException("Study not found in BrAPI service: " + studyDbId); + } + BrAPIStudy study = studies.get(0); + if (StringUtils.isBlank(study.getTrialDbId())) { + throw new IllegalStateException("TrialDbId is not set for an existing Study: " + study.getStudyDbId()); + } + trialDbId = study.getTrialDbId(); + + return trialDbId; + } + /** * This method processes an existing trial, retrieves the experiment ID from the trial's external references, * and caches the trial with the corresponding experiment ID in a map. @@ -149,6 +263,28 @@ private void processAndCacheTrial( new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } + /** + * Constructs a PendingImportObject containing a BrAPITrial object based on the input BrAPITrial. + * + * This function takes a BrAPITrial object as input and constructs a PendingImportObject which + * encapsulates the trial along with its associated experiment ID. The experiment ID is retrieved + * from the external references of the trial object using utility method getExternalReference. + * If the experiment ID is not found in the external references, an InternalServerException is thrown. + * + * @param trial the BrAPITrial object for which the PendingImportObject is to be constructed + * @return a PendingImportObject containing the BrAPITrial object and its associated experiment ID + * @throws InternalServerException if the experiment ID is not found in the external references of the trial + */ + public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial trial) throws InternalServerException { + PendingImportObject pio = null; + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(trial.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) + .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId)); + return pio; + } + /** * Initializes trials by name without scope for the given program. * From 4b8960bbe85f53b6c975d2e1fc624df4a9fafa9f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 16:16:06 -0400 Subject: [PATCH 37/61] make required studies middleware --- .../read/brapi/RequiredBrAPIData.java | 9 ++- .../read/brapi/RequiredStudies.java | 78 +++++++++++++++++++ .../middleware/read/brapi/RequiredTrials.java | 8 +- .../experiment/service/StudyService.java | 73 +++++++++++++++++ .../experiment/service/TrialService.java | 2 +- 5 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 539f63ca7..aef3d5f9c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -10,11 +10,14 @@ @Slf4j public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - Provider existingObservationUnitsProvider; + Provider requiredObservationUnitsProvider; + Provider requiredTrialsProvider; @Inject - public RequiredBrAPIData(Provider existingObservationUnitsProvider) { - this.middleware.link(existingObservationUnitsProvider.get()); + public RequiredBrAPIData(Provider requiredObservationUnitsProvider, + Provider requiredTrialsProvider) { + this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids + requiredTrialsProvider.get()); // Fetch the BrAPI trials belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java new file mode 100644 index 000000000..5803d07f0 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -0,0 +1,78 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredStudies extends ExpUnitMiddleware { + StudyService studyService; + + @Inject + public RequiredStudies(StudyService studyService) { + this.studyService = studyService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Map> pendingUnitByNameNoScope; + Set studyDbIds; + List brAPIStudies; + List> pendingStudies; + Map> pendingStudyByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no required units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service studies belonging to required units"); + + // Get the dbIds of the studies belonging to the required exp units + studyDbIds = pendingUnitByNameNoScope.values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); + + // Get the BrAPI studies belonging to required exp units + brAPIStudies = studyDbIds.stream().map(dbId -> { + BrAPIStudy study = null; + try { + study = studyService.fetchBrapiStudyByDbId(dbId, program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return study; + }).collect(Collectors.toList()); + + // Construct the pending studies from the BrAPI trials + pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending study by study name with the program key removed + pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index 3761a5a7d..732c662eb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -1,15 +1,11 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -48,8 +44,8 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get the dbIds of the trials belonging to the required exp units trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); - // Get the trials belonging to required exp units - brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialsBelongingToUnit(dbId, program)).collect(Collectors.toList()); + // Get the BrAPI trials belonging to required exp units + brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialBelongingToUnit(dbId, program)).collect(Collectors.toList()); // Construct the pending trials from the BrAPI trials pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 2356855ba..dce3571b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -77,6 +77,36 @@ public PendingImportObject processAndCacheStudy( return pendingStudy; } + /** + * Constructs a PendingImportObject containing a BrAPIStudy object based on the provided BrAPIStudy and Program. + * This function retrieves the external reference for the study and maps the season dbid to the corresponding year. + * + * @param brAPIStudy The BrAPIStudy object to construct the PendingImportObject from. + * @param program The Program object associated with the study. + * @return A PendingImportObject containing the formatted BrAPIStudy object. + * @throws IllegalStateException If the external reference for the study is not found. + */ + public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brAPIStudy, Program program) { + // Retrieve external reference for the study + BrAPIExternalReference xref = Utilities.getExternalReference(brAPIStudy.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId()); + + // Map season dbid to year + String seasonDbId = brAPIStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = seasonDbIdToYear(seasonDbId, program.getId()); + brAPIStudy.setSeasons(Collections.singletonList(seasonYear)); + } + + // Create and return a PendingImportObject for the BrAPIStudy + return new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(brAPIStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + } + // TODO: used by both workflows private String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; @@ -127,6 +157,49 @@ public List fetchStudiesByDbId(Set studyDbIds, Program progr return studies; } + /** + * Fetches a BrAPI study by its unique database identifier and program. + * This method retrieves a BrAPI study identified by the given study database identifier + * and corresponding program. + * + * @param studyDbId A String representing the unique database identifier of the study. + * @param program The Program object specifying the program to which the study belongs. + * @return The BrAPIStudy object representing the study fetched from the database. + * @throws ApiException if the study with the provided studyDbId is not found. + */ + public BrAPIStudy fetchBrapiStudyByDbId(String studyDbId, Program program) throws ApiException { + BrAPIStudy study = null; // Initializing the study object + List studies = brAPIStudyDAO.getStudiesByStudyDbId(Set.of(studyDbId), program); // Retrieving studies from the database + + if (studies.size() == 0) { + throw new IllegalStateException( + "Study not found for studyDbId: " + studyDbId); // Throwing exception if no study is found for provided studyDbId + } + + return studies.get(0); // Returning the first study from the list + } + + /** + * Retrieves the study database ID belonging to a pending unit in BrAPI format. + * + * This method takes a PendingImportObject containing a BrAPIObservationUnit + * object and returns the study database ID associated with the unit, if it exists. + * + * @param pio The PendingImportObject containing the BrAPIObservationUnit object for which the study database ID is to be retrieved. + * @return The study database ID belonging to the pending unit, or null if the unit does not exist or if the study database ID is not set. + */ + public String getStudyDbIdBelongingToPendingUnit(PendingImportObject pio) { + String studyDbId = null; + + // Check if the BrAPI object in the PendingImportObject is not null + if (pio.getBrAPIObject() != null) { + // Retrieve the study database ID from the BrAPIObservationUnit object + studyDbId = pio.getBrAPIObject().getStudyDbId(); + } + + return studyDbId; + } + // TODO: used by both workflows private void initializeStudiesForExistingObservationUnits( Program program, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index d768178ba..86ba933b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -121,7 +121,7 @@ public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, * @throws InternalServerException If an internal server error occurs while processing the request. * @throws IllegalStateException If the trial with the specified trialDbId is not found. */ - public BrAPITrial fetchBrapiTrialsBelongingToUnit(String trialDbId, Program program) throws InternalServerException { + public BrAPITrial fetchBrapiTrialBelongingToUnit(String trialDbId, Program program) throws InternalServerException { try { List trials = brAPITrialDAO.getTrialsByDbIds(Set.of(trialDbId), program); BrAPITrial trial = trials.get(0); From 71c145eef9940536acf3531c994dd771058dc77f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 May 2024 17:54:53 -0400 Subject: [PATCH 38/61] create required locations middleware --- .../read/brapi/RequiredBrAPIData.java | 7 +- .../read/brapi/RequiredLocations.java | 77 +++++++++++++++++++ .../experiment/service/LocationService.java | 45 ++++++++++- 3 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index aef3d5f9c..83b15dcae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -12,12 +12,15 @@ public class RequiredBrAPIData extends ExpUnitMiddleware { ExpUnitMiddleware middleware; Provider requiredObservationUnitsProvider; Provider requiredTrialsProvider; + Provider requiredStudiesProvider; @Inject public RequiredBrAPIData(Provider requiredObservationUnitsProvider, - Provider requiredTrialsProvider) { + Provider requiredTrialsProvider, + Provider requiredStudiesProvider) { this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get()); // Fetch the BrAPI trials belonging to the exp units + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get()); // Fetch the BrAPI studies belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java new file mode 100644 index 000000000..49cdded53 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -0,0 +1,77 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredLocations extends ExpUnitMiddleware { + LocationService locationService; + + @Inject + public RequiredLocations(LocationService locationService) { + this.locationService = locationService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set locationDbIds; + List brapiLocations; + List> pendingLocations; + Map> pendingStudyByNameNoScope; + Map> pendingLocationByName; + + program = context.getImportContext().getProgram(); + pendingStudyByNameNoScope = context.getPendingData().getStudyByNameNoScope(); + + // nothing to do if there are no required units + if (pendingStudyByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, locations belonging to required units"); + + // Get the dbIds of the studies belonging to the required exp units + locationDbIds = pendingStudyByNameNoScope.values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + + // Get the locations belonging to required exp units + brapiLocations = locationDbIds.stream().map(dbId -> { + ProgramLocation location = null; + try { + location = locationService.fetchLocationByDbId(dbId, program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + return location; + }).collect(Collectors.toList()); + + // Construct the pending locations from the BrAPI locations + pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending location by location name + pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setLocationByName(pendingLocationByName); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index 624d02842..cbc993bb5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -17,6 +17,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramLocation; +import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.utilities.Utilities; import javax.inject.Singleton; @@ -26,6 +27,48 @@ @Singleton @Slf4j public class LocationService { + private final ProgramLocationService programLocationService; + public LocationService(ProgramLocationService programLocationService) { + this.programLocationService = programLocationService; + } + + /** + * Fetches a ProgramLocation based on the provided locationDbId within a given program. + * + * @param locationDbId the unique identifier of the location + * @param program the program in which the location is to be fetched + * @return the ProgramLocation associated with the given locationDbId in the program + * @throws ApiException if the program location cannot be found for the specified locationDbId + */ + public ProgramLocation fetchLocationByDbId(String locationDbId, Program program) throws ApiException { + ProgramLocation programLocation = null; // Initializing the ProgramLocation object + + // Retrieving locations based on the locationDbId and the program's ID + List locations = programLocationService.getLocationsByDbId(List.of(locationDbId), program.getId()); + + // If no locations are found, throw an IllegalStateException with an error message + if (locations.size() == 0) { + throw new IllegalStateException( + "Location not found for locationDbId: " + locationDbId); // Throwing exception if no location is found for provided locationDbId + } + + // Set the programLocation to the first location found in the list of locations + programLocation = locations.get(0); + + // Return the fetched ProgramLocation + return programLocation; + } + + /** + * Constructs a PendingImportObject of type ProgramLocation from a given BrAPI ProgramLocation. + * This method creates a new PendingImportObject with the state set to EXISTING and the BrAPI ProgramLocation as the source object. + * + * @param brapiLocation The BrAPI ProgramLocation from which the PendingImportObject should be constructed + * @return PendingImportObject The PendingImportObject created from the BrAPI ProgramLocation + */ + public PendingImportObject constructPIOFromBrapiLocation(ProgramLocation brapiLocation) { + return new PendingImportObject<>(ImportObjectState.EXISTING, brapiLocation, brapiLocation.getId()); + } // used by expunit workflow public Map> initializeLocationByName( @@ -41,7 +84,7 @@ public Map> initializeLocationByNam .getLocationDbId()) .collect(Collectors.toSet()); try { - existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + existingLocations.addAll(programLocationService.getLocationsByDbId(locationDbIds, program.getId())); } catch (ApiException e) { log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); From 5635dcf00ad552969eb4a342dc9c0d6afb20cb73 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 May 2024 14:33:30 -0400 Subject: [PATCH 39/61] add required dataset middleware --- .../AppendOverwritePhenotypesWorkflow.java | 1 + .../read/brapi/RequiredBrAPIData.java | 13 ++- .../read/brapi/RequiredDatasets.java | 80 +++++++++++++++++++ .../experiment/middleware/Middleware.java | 10 ++- .../experiment/service/DatasetService.java | 74 +++++++++++++++++ 5 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 125b52d14..b97e40536 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -25,6 +25,7 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { + this.middleware.link(handleErrProvider.get(), validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 83b15dcae..4cfe2a5ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -13,14 +13,21 @@ public class RequiredBrAPIData extends ExpUnitMiddleware { Provider requiredObservationUnitsProvider; Provider requiredTrialsProvider; Provider requiredStudiesProvider; + Provider requiredLocationsProvider; + Provider requiredDatasetsProvider; @Inject public RequiredBrAPIData(Provider requiredObservationUnitsProvider, Provider requiredTrialsProvider, - Provider requiredStudiesProvider) { + Provider requiredStudiesProvider, + Provider requiredLocationsProvider, + Provider requiredDatasetsProvider) { + this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units - requiredStudiesProvider.get()); // Fetch the BrAPI studies belonging to the exp units + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units + requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units + requiredDatasetsProvider.get()); // Fetch the datasets belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java new file mode 100644 index 000000000..ccba81b42 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -0,0 +1,80 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredDatasets extends ExpUnitMiddleware { + private final DatasetService datasetService; + + @Inject + public RequiredDatasets(DatasetService datasetService) { + this.datasetService = datasetService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + String datasetId; + BrAPIListDetails dataset = null; + PendingImportObject pendingDataset; + Map> pendingTrialByNameNoScope; + Map> pendingDatasetByName; + + program = context.getImportContext().getProgram(); + pendingTrialByNameNoScope = context.getPendingData().getTrialByNameNoScope(); + + // nothing to do if there are no trials with dataset ids + if (pendingTrialByNameNoScope.size() == 0 || + !pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + return processNext(context); + } + log.debug("fetching from BrAPI service, datasets belonging to required units"); + + // Get the dbId of the trial belonging to the required exp units + datasetId = pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + + // Get the dataset belonging to required exp units + try { + dataset = datasetService.fetchDatasetById(datasetId, program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + // Construct the pending locations from the BrAPI locations + pendingDataset = datasetService.constructPIOFromDataset(dataset, program); + + // Construct a hashmap to look up the pending dataset by dataset name + pendingDatasetByName = new HashMap<>(); + pendingDatasetByName.put(dataset.getListName(), pendingDataset); + + // Add the map to the context for use in processing import + context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index 6d097722f..cee2a4c15 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -10,12 +10,12 @@ public abstract class Middleware { /** * Builds chains of middleware objects. */ - public static Middleware link(Middleware first, Middleware... chain) { + public Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { nextInChain.prior = head.getLastLink(); head.getLastLink().next = nextInChain; - head = nextInChain.getLastLink(); + head = nextInChain; } return first; } @@ -38,7 +38,7 @@ protected boolean processNext(T context) { } return next.process(context); } - + /** * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. @@ -53,4 +53,8 @@ protected boolean compensatePrior(T context, MiddlewareError error) { private Middleware getLastLink() { return this.next == null ? this : this.next.getLastLink(); } + + private Middleware getFirstLink() { + return this.prior == null ? this : this.prior.getFirstLink(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index e8c37983d..70169478e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -1,13 +1,16 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -21,10 +24,81 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; +import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; public class DatasetService { + private final BrAPIListDAO brAPIListDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + @Inject + public DatasetService(BrAPIListDAO brapiListDAO) { + this.brAPIListDAO = brapiListDAO; + } + /** + * Module: Dataset Utility + * + * This module provides utility functions for interacting with datasets using the BrAPI standards. + * It includes methods for fetching dataset details, creating new datasets, updating existing datasets, etc. + * Usage: This module can be used in various applications where handling BrAPI-compliant datasets is required. + */ + + /** + * Fetches dataset details by dataset ID and program + * + * This function fetches details of a dataset by its ID and associated program from a data source using the BrAPI standards. + * + * @param id The unique identifier of the dataset to fetch + * @param program The program object associated with the dataset + * @return BrAPIListDetails object containing the details of the dataset + * @throws ApiException if there is an issue with fetching the dataset details from the data source + */ + public BrAPIListDetails fetchDatasetById(String id, Program program) throws ApiException { + BrAPIListDetails dataSetDetails = null; + + // Retrieve existing dataset summaries based on program ID and external reference + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(id)); + + // Check if the existing dataset summaries are returned, throw exception if not + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("Existing dataset summary not returned from BrAPI server"); + } + + // Retrieve dataset details using the list DB ID from the existing dataset summary + dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + + return dataSetDetails; + } + + /** + * Constructs a PendingImportObject for a BrAPIListDetails dataset. + * This method retrieves the external reference for the dataset from the existing list + * based on a specific reference source. It then creates a PendingImportObject for the dataset + * with the existing list and reference ID. + * + * @param dataset The BrAPIListDetails dataset for which to construct the PendingImportObject + * @param program + * @return A PendingImportObject containing the dataset with the existing list and reference ID + * @throws IllegalStateException if external references weren't found for the list + */ + public PendingImportObject constructPIOFromDataset(BrAPIListDetails dataset, Program program) { + // Get the external reference for the dataset from the existing list + BrAPIExternalReference xref = Utilities.getExternalReference(dataset.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId()); + + // Create a PendingImportObject for the dataset with the existing list and reference ID + return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); + } + // TODO: used by expunit worflow public Map> initializeObsVarDatasetForExistingObservationUnits( Map> trialByName, From b0198f040757d81cb945e27a490605b03a0c0e71 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 May 2024 16:12:00 -0400 Subject: [PATCH 40/61] create required germplasm middleware --- .../read/brapi/RequiredBrAPIData.java | 7 +- .../read/brapi/RequiredDatasets.java | 9 +-- .../read/brapi/RequiredGermplasm.java | 73 +++++++++++++++++++ .../experiment/middleware/Middleware.java | 2 +- .../experiment/service/GermplasmService.java | 60 +++++++++++++++ 5 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 4cfe2a5ca..2446c9358 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -15,19 +15,22 @@ public class RequiredBrAPIData extends ExpUnitMiddleware { Provider requiredStudiesProvider; Provider requiredLocationsProvider; Provider requiredDatasetsProvider; + Provider requiredGermplasmProvider; @Inject public RequiredBrAPIData(Provider requiredObservationUnitsProvider, Provider requiredTrialsProvider, Provider requiredStudiesProvider, Provider requiredLocationsProvider, - Provider requiredDatasetsProvider) { + Provider requiredDatasetsProvider, + Provider requiredGermplasmProvider) { this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units - requiredDatasetsProvider.get()); // Fetch the datasets belonging to the exp units + requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units + requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java index ccba81b42..7e6967bc9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -2,7 +2,6 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; @@ -11,15 +10,11 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.LocationService; import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; import javax.inject.Inject; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Slf4j public class RequiredDatasets extends ExpUnitMiddleware { @@ -49,7 +44,7 @@ public boolean process(ExpUnitMiddlewareContext context) { } log.debug("fetching from BrAPI service, datasets belonging to required units"); - // Get the dbId of the trial belonging to the required exp units + // Get the id of the dataset belonging to the required exp units datasetId = pendingTrialByNameNoScope.values().iterator().next().getBrAPIObject() .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) @@ -65,7 +60,7 @@ public boolean process(ExpUnitMiddlewareContext context) { })); } - // Construct the pending locations from the BrAPI locations + // Construct the pending dataset from the BrAPI observation variable list pendingDataset = datasetService.constructPIOFromDataset(dataset, program); // Construct a hashmap to look up the pending dataset by dataset name diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java new file mode 100644 index 000000000..fe8dec94c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -0,0 +1,73 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.GermplasmService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RequiredGermplasm extends ExpUnitMiddleware { + private final GermplasmService germplasmService; + + @Inject + public RequiredGermplasm(GermplasmService germplasmService) { + + this.germplasmService = germplasmService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set germplasmDbId; + List brapiGermplasm = null; + List> pendingGermplasm; + Map> pendingGermplasmByGID; + Map> pendingUnitByNameNoScope; + + program = context.getImportContext().getProgram(); + pendingUnitByNameNoScope = context.getPendingData().getObservationUnitByNameNoScope(); + + // nothing to do if there are no observation units + if (pendingUnitByNameNoScope.size() == 0) { + return processNext(context); + } + log.debug("fetching from BrAPI service, germplasm belonging to required units"); + + // Get the dbIds of the germplasm belonging to the required exp units + Set germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + + // Get the dataset belonging to required exp units + try { + brapiGermplasm = germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), program); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + // Construct the pending germplasm from the BrAPI locations + pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index cee2a4c15..108d6cd88 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -38,7 +38,7 @@ protected boolean processNext(T context) { } return next.process(context); } - + /** * Runs the compensating local transaction for the prior local transaction or ends traversing if * we're at the first local transaction of the transaction. diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java index 386b96125..b4e7127cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/GermplasmService.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -7,6 +8,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -17,6 +19,64 @@ import java.util.stream.Collectors; public class GermplasmService { + private final BrAPIGermplasmDAO germplasmDAO; + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + + public GermplasmService(BrAPIGermplasmDAO germplasmDAO) { + this.germplasmDAO = germplasmDAO; + } + + /** + * Retrieves a list of BrAPI Germplasm objects based on the provided set of database IDs and a Program object. + * + * @param dbIds A Set of database IDs (strings) used to filter germplasm data retrieval. + * @param program The Program object representing the program associated with the germplasm data. + * @return A List of BrAPIGermplasm objects that match the provided database IDs and program. + * @throws ApiException If an error occurs during the retrieval process. + */ + public List fetchGermplasmByDbId(Set dbIds, Program program) throws ApiException { + List brapiGermplasm = null; + brapiGermplasm = germplasmDAO.getGermplasmsByDBID(dbIds, program.getId()); + return brapiGermplasm; + } + + /** + * This method constructs a PendingImportObject for a given BrAPI Germplasm. + * It retrieves the External Reference associated with the Germplasm and constructs a PendingImportObject with ImportObjectState set to EXISTING. + * + * @param brapiGermplasm The BrAPI Germplasm object for which the PendingImportObject needs to be constructed + * @return PendingImportObject A PendingImportObject containing the BrAPI Germplasm object and its External Reference + * @throws IllegalStateException if the External Reference for the Germplasm is not found + */ + public PendingImportObject constructPIOFromBrapiGermplasm(BrAPIGermplasm brapiGermplasm) { + // Initialize the PendingImportObject to null + PendingImportObject pio = null; + + // Retrieve the External Reference associated with the Germplasm from the Utilities class + BrAPIExternalReference xref = Utilities.getExternalReference(brapiGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + // Throw an exception if External Reference is not found + .orElseThrow(() -> new IllegalStateException("External references weren't found for germplasm (dbid): " + brapiGermplasm.getGermplasmDbId())); + + // Construct the PendingImportObject with ImportObjectState set to EXISTING and External Reference UUID + pio = new PendingImportObject<>(ImportObjectState.EXISTING, brapiGermplasm, UUID.fromString(xref.getReferenceId())); + + return pio; + } + + /** + * Retrieves the Germplasm ID from a PendingImportObject containing BrAPI Germplasm data. + * This method extracts the Germplasm ID (GID) from the Accession Number of the BrAPI Germplasm object within the PendingImportObject. + * + * @param pio a PendingImportObject that wraps BrAPI Germplasm data + * @return a String representing the Germplasm ID extracted from the Accession Number + */ + public String getGIDFromGermplasmPIO(PendingImportObject pio) { + String gid = null; + gid = pio.getBrAPIObject().getAccessionNumber(); + return gid; + } + // TODO: used by expunit workflow public Map> initializeGermplasmByGIDForExistingObservationUnits( Map> unitByName, From 61c1229b654dbee8455925ba5da4dd91099a6eff Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 May 2024 17:11:24 -0400 Subject: [PATCH 41/61] batch location request --- .../experiment/ExperimentUtilities.java | 2 +- .../read/brapi/RequiredLocations.java | 33 ++++++++----------- .../experiment/service/LocationService.java | 32 +++++++++--------- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index d22f9ae2f..2a71177ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -22,7 +22,7 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; - + public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java index 49cdded53..fed9bf455 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredLocations.java @@ -49,28 +49,23 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get the dbIds of the studies belonging to the required exp units locationDbIds = pendingStudyByNameNoScope.values().stream().map(pio -> pio.getBrAPIObject().getLocationDbId()).collect(Collectors.toSet()); + try { + // Get the locations belonging to required exp units + brapiLocations = locationService.fetchLocationsByDbId(locationDbIds, program); - // Get the locations belonging to required exp units - brapiLocations = locationDbIds.stream().map(dbId -> { - ProgramLocation location = null; - try { - location = locationService.fetchLocationByDbId(dbId, program); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - return location; - }).collect(Collectors.toList()); + // Construct the pending locations from the BrAPI locations + pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); - // Construct the pending locations from the BrAPI locations - pendingLocations = brapiLocations.stream().map(locationService::constructPIOFromBrapiLocation).collect(Collectors.toList()); + // Construct a hashmap to look up the pending location by location name + pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); - // Construct a hashmap to look up the pending location by location name - pendingLocationByName = pendingLocations.stream().collect(Collectors.toMap(pio -> pio.getBrAPIObject().getName(), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setLocationByName(pendingLocationByName); + // Add the map to the context for use in processing import + context.getPendingData().setLocationByName(pendingLocationByName); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java index cbc993bb5..2fe6815ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/LocationService.java @@ -24,6 +24,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + @Singleton @Slf4j public class LocationService { @@ -33,30 +35,28 @@ public LocationService(ProgramLocationService programLocationService) { } /** - * Fetches a ProgramLocation based on the provided locationDbId within a given program. + * Fetches a list of ProgramLocation objects based on the provided locationDbIds and program ID. * - * @param locationDbId the unique identifier of the location - * @param program the program in which the location is to be fetched - * @return the ProgramLocation associated with the given locationDbId in the program - * @throws ApiException if the program location cannot be found for the specified locationDbId + * @param locationDbIds A set of locationDbIds used to query locations. + * @param program The Program object for which locations are being fetched. + * @return A list of ProgramLocation objects matching the given locationDbIds for the program. + * @throws ApiException if there is an issue with fetching the locations or if any location(s) are not found. */ - public ProgramLocation fetchLocationByDbId(String locationDbId, Program program) throws ApiException { - ProgramLocation programLocation = null; // Initializing the ProgramLocation object + public List fetchLocationsByDbId(Set locationDbIds, Program program) throws ApiException { + List programLocations = null; // Initializing the ProgramLocations list // Retrieving locations based on the locationDbId and the program's ID - List locations = programLocationService.getLocationsByDbId(List.of(locationDbId), program.getId()); + programLocations = programLocationService.getLocationsByDbId(locationDbIds, program.getId()); // If no locations are found, throw an IllegalStateException with an error message - if (locations.size() == 0) { - throw new IllegalStateException( - "Location not found for locationDbId: " + locationDbId); // Throwing exception if no location is found for provided locationDbId + if (locationDbIds.size() != programLocations.size()) { + Set missingIds = new HashSet<>(locationDbIds); + missingIds.removeAll(programLocations.stream().map(ProgramLocation::getLocationDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Location not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); } - // Set the programLocation to the first location found in the list of locations - programLocation = locations.get(0); - - // Return the fetched ProgramLocation - return programLocation; + // Return the fetched ProgramLocations + return programLocations; } /** From c69762251b376d74714105e8789bb27a226af7f6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 10 May 2024 10:47:54 -0400 Subject: [PATCH 42/61] batch requests for studies and trials --- .../experiment/ExperimentUtilities.java | 2 +- .../read/brapi/RequiredGermplasm.java | 25 +++++---- .../read/brapi/RequiredObservationUnits.java | 16 ++---- .../read/brapi/RequiredStudies.java | 33 +++++------- .../middleware/read/brapi/RequiredTrials.java | 27 ++++++---- .../service/ObservationUnitService.java | 40 +++++++++++---- .../experiment/service/StudyService.java | 35 +++++++------ .../experiment/service/TrialService.java | 51 ++++++++++++------- 8 files changed, 133 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index 2a71177ed..d22f9ae2f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -22,7 +22,7 @@ public class ExperimentUtilities { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; - + public static List importRowsToExperimentObservations(List importRows) { return importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java index fe8dec94c..1d40ea781 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredGermplasm.java @@ -32,7 +32,7 @@ public RequiredGermplasm(GermplasmService germplasmService) { @Override public boolean process(ExpUnitMiddlewareContext context) { Program program; - Set germplasmDbId; + Set germplasmDbIds; List brapiGermplasm = null; List> pendingGermplasm; Map> pendingGermplasmByGID; @@ -48,26 +48,25 @@ public boolean process(ExpUnitMiddlewareContext context) { log.debug("fetching from BrAPI service, germplasm belonging to required units"); // Get the dbIds of the germplasm belonging to the required exp units - Set germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); - - // Get the dataset belonging to required exp units + germplasmDbIds = pendingUnitByNameNoScope.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); try { + // Get the dataset belonging to required exp units brapiGermplasm = germplasmService.fetchGermplasmByDbId(new HashSet<>(germplasmDbIds), program); + + // Construct the pending germplasm from the BrAPI locations + pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending germplasm by gid + pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); } catch (ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); } - // Construct the pending germplasm from the BrAPI locations - pendingGermplasm = brapiGermplasm.stream().map(germplasmService::constructPIOFromBrapiGermplasm).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending germplasm by gid - pendingGermplasmByGID = pendingGermplasm.stream().collect(Collectors.toMap(germplasmService::getGIDFromGermplasmPIO, pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setExistingGermplasmByGID(pendingGermplasmByGID); - return processNext(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index d2e6dab21..5a3fa8301 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -36,20 +36,12 @@ public boolean process(ExpUnitMiddlewareContext context) { log.debug("fetching required exp units from BrAPI service"); program = context.getImportContext().getProgram(); - try { - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getReferenceUnits(new HashSet<>(expUnitIds), program); - if (brapiUnits.size() != expUnitIds.size()) { - - // Handle case of missing Observation Units in data store - missingIds = observationUnitService.collectMissingOUIds(new HashSet<>(expUnitIds), new ArrayList<>(brapiUnits)); - this.compensate(context, new MiddlewareError(() -> { - throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingIds)); - })); - } + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); // Construct pending import objects from the units pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java index 5803d07f0..493ae0021 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredStudies.java @@ -50,28 +50,23 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get the dbIds of the studies belonging to the required exp units studyDbIds = pendingUnitByNameNoScope.values().stream().map(studyService::getStudyDbIdBelongingToPendingUnit).collect(Collectors.toSet()); - // Get the BrAPI studies belonging to required exp units - brAPIStudies = studyDbIds.stream().map(dbId -> { - BrAPIStudy study = null; - try { - study = studyService.fetchBrapiStudyByDbId(dbId, program); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - return study; - }).collect(Collectors.toList()); + try { + // Get the BrAPI studies belonging to required exp units + brAPIStudies = studyService.fetchBrapiStudiesByDbId(studyDbIds, program); - // Construct the pending studies from the BrAPI trials - pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); + // Construct the pending studies from the BrAPI trials + pendingStudies = brAPIStudies.stream().map(pio -> studyService.constructPIOFromBrapiStudy(pio, program)).collect(Collectors.toList()); - // Construct a hashmap to look up the pending study by study name with the program key removed - pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); - - // Add the map to the context for use in processing import - context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + // Construct a hashmap to look up the pending study by study name with the program key removed + pendingStudyByNameNoScope = pendingStudies.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKeyAndUnknownAdditionalData(pio.getBrAPIObject().getStudyName(), program.getKey()), pio -> pio)); + // Add the map to the context for use in processing import + context.getPendingData().setStudyByNameNoScope(pendingStudyByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java index 732c662eb..aac07d77c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredTrials.java @@ -1,11 +1,13 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.read.brapi; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.TrialService; import org.breedinginsight.model.Program; import org.breedinginsight.utilities.Utilities; @@ -39,22 +41,29 @@ public boolean process(ExpUnitMiddlewareContext context) { if (pendingUnitByNameNoScope.size() == 0) { return processNext(context); } - log.debug("fetching from BrAPI service trials belonging to required units"); + log.debug("fetching from BrAPI service, trials belonging to required units"); // Get the dbIds of the trials belonging to the required exp units trialDbIds = pendingUnitByNameNoScope.values().stream().map(pendingUnit -> trialService.getTrialDbIdBelongingToPendingUnit(pendingUnit, program)).collect(Collectors.toSet()); + try { + // Get the BrAPI trials belonging to required exp units + brAPITrials = trialService.fetchBrapiTrialsByDbId(trialDbIds, program); - // Get the BrAPI trials belonging to required exp units - brAPITrials = trialDbIds.stream().map(dbId -> trialService.fetchBrapiTrialBelongingToUnit(dbId, program)).collect(Collectors.toList()); + // Construct the pending trials from the BrAPI trials + pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); - // Construct the pending trials from the BrAPI trials - pendingTrials = brAPITrials.stream().map(trialService::constructPIOFromBrapiTrial).collect(Collectors.toList()); + // Construct a hashmap to look up the pending trial by trial name with the program key removed + pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); + + // Add the map to the context for use in processing import + context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } - // Construct a hashmap to look up the pending trial by trial name with the program key removed - pendingTrialByNameNoScope = pendingTrials.stream().collect(Collectors.toMap(pio -> Utilities.removeProgramKey(pio.getBrAPIObject().getTrialName(), program.getKey()), pio -> pio)); - // Add the map to the context for use in processing import - context.getPendingData().setTrialByNameNoScope(pendingTrialByNameNoScope); return processNext(context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index d7f3708d5..e19c196d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; @@ -29,6 +30,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + public class ObservationUnitService { private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; @Property(name = "brapi.server.reference-source") @@ -40,19 +43,36 @@ public ObservationUnitService(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { } /** - * Retrieves the reference Observation Units based on the provided set of experimental unit IDs and the associated Program. - * This function queries the database to fetch Observation Units that match the specified IDs within the given Program. + * Retrieves a list of BrAPI (Breeding API) observation units by their database IDs for a given set of experimental unit IDs and program. + * + * This method queries the BrAPIObservationUnitDAO to retrieve BrAPI observation units based on the provided experimental unit IDs and program. + * If the database IDs of the retrieved BrAPI observation units do not match the provided experimental unit IDs, an IllegalStateException is thrown. + * The exception includes information on the missing observation unit database IDs. * - * @param expUnitIds A set of unique identifiers representing the Experimental Units for which reference Observation Units are required. - * These IDs serve as filters to determine the relevant Observation Units. - * @param program The Program to which the Observation Units belong. This parameter ensures that the fetched units are specific to the designated Program. - * @return A List of BrAPIObservationUnit objects that correspond to the reference Observation Units matching the provided expUnitIds within the given Program. - * @throws ApiException If an error occurs during the retrieval process, an ApiException is thrown to handle exceptional scenarios. + * @param expUnitIds a set of experimental unit IDs for which to retrieve BrAPI observation units + * @param program the program for which to retrieve BrAPI observation units + * @return a list of BrAPIObservationUnit objects corresponding to the provided experimental unit IDs + * @throws ApiException if an error occurs during the retrieval of observation units + * @throws IllegalStateException if the retrieved observation units do not match the provided experimental unit IDs */ - public List getReferenceUnits(Set expUnitIds, - Program program) throws ApiException { + public List getObservationUnitsByDbId(Set expUnitIds, Program program) throws ApiException, IllegalStateException { + List brapiUnits = null; + // Retrieve reference Observation Units based on IDs - return brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(expUnitIds), program); + brapiUnits = brAPIObservationUnitDAO.getObservationUnitsById(expUnitIds, program); + + // If no BrAPI units are found, throw an IllegalStateException with an error message + if (expUnitIds.size() != brapiUnits.size()) { + Set missingIds = new HashSet<>(expUnitIds); + + // Calculate missing IDs based on retrieved BrAPI units + missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); + + // Throw exception with missing IDs information + throw new IllegalStateException("Observation unit not found for unit dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return brapiUnits; } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index dce3571b6..baf646689 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -33,6 +33,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + @Singleton @Slf4j public class StudyService { @@ -158,25 +160,28 @@ public List fetchStudiesByDbId(Set studyDbIds, Program progr } /** - * Fetches a BrAPI study by its unique database identifier and program. - * This method retrieves a BrAPI study identified by the given study database identifier - * and corresponding program. + * Fetch BrAPI studies by their database identifiers for a given program. + * + * This method retrieves a list of BrAPI studies based on the provided set of study database identifiers + * and a specified program. It utilizes the BrAPIStudyDAO to fetch studies from the database. * - * @param studyDbId A String representing the unique database identifier of the study. - * @param program The Program object specifying the program to which the study belongs. - * @return The BrAPIStudy object representing the study fetched from the database. - * @throws ApiException if the study with the provided studyDbId is not found. + * @param studyDbIds A set of study database identifiers for filtering the studies. + * @param program The program related to the studies. + * @return A list of BrAPIStudy objects representing the fetched studies. + * @throws ApiException If there are issues in retrieving studies or if any study database identifier is missing. */ - public BrAPIStudy fetchBrapiStudyByDbId(String studyDbId, Program program) throws ApiException { - BrAPIStudy study = null; // Initializing the study object - List studies = brAPIStudyDAO.getStudiesByStudyDbId(Set.of(studyDbId), program); // Retrieving studies from the database - - if (studies.size() == 0) { - throw new IllegalStateException( - "Study not found for studyDbId: " + studyDbId); // Throwing exception if no study is found for provided studyDbId + public List fetchBrapiStudiesByDbId(Set studyDbIds, Program program) throws ApiException { + List brapiStudies = null; // Initializing the study object + brapiStudies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); // Retrieving studies from the database + + // If no studies are found, throw an IllegalStateException with an error message + if (studyDbIds.size() != brapiStudies.size()) { + Set missingIds = new HashSet<>(studyDbIds); + missingIds.removeAll(brapiStudies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toSet())); + throw new IllegalStateException("Study not found for location dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); } - return studies.get(0); // Returning the first study from the list + return brapiStudies; } /** diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 86ba933b5..430153d40 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -20,6 +20,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramLocation; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; @@ -28,6 +29,8 @@ import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.COMMA_DELIMITER; + @Singleton @Slf4j public class TrialService { @@ -44,6 +47,12 @@ public TrialService(BrAPITrialDAO brAPITrialDAO, this.brAPITrialDAO = brAPITrialDAO; this.studyService = studyService; } + /** + * Module: BrAPITrialService + * Description: This module contains methods for retrieving BrAPI trials based on trial database IDs and programs. + * brAPITrialDAO: Data Access Object for interacting with BrAPI trials in the database. + * fetchBrapiTrialsByDbId: Method to fetch BrAPI trials based on provided trial database IDs and program. + */ /** * Retrieves the TrialDbId belonging to a pending unit based on the provided BrAPI observation unit and program. @@ -113,28 +122,36 @@ public List fetchBrapiTrialsBelongingToUnits(Set trialDbIds, } /** - * Fetches a BrAPI trial belonging to a specific unit based on the provided trialDbId and program. + * Retrieves a list of BrAPI trials based on a set of trial database IDs and a specified program. * - * @param trialDbId The unique identifier of the trial to be fetched. - * @param program The program to which the trial belongs. - * @return The BrAPI trial belonging to the specified unit. - * @throws InternalServerException If an internal server error occurs while processing the request. - * @throws IllegalStateException If the trial with the specified trialDbId is not found. + * @param trialDbIds a set of trial database IDs used to retrieve the BrAPI trials + * @param program the program associated with the trials + * @return a list of BrAPITrial objects that match the provided trial database IDs and program + * @throws InternalServerException if there is an internal server error during the retrieval process + * @throws ApiException if there is an exception while fetching the trials */ - public BrAPITrial fetchBrapiTrialBelongingToUnit(String trialDbId, Program program) throws InternalServerException { - try { - List trials = brAPITrialDAO.getTrialsByDbIds(Set.of(trialDbId), program); - BrAPITrial trial = trials.get(0); - if (trials.size() == 0) { - throw new IllegalStateException("Trial not found for trialDbId(s): " + trialDbId); - } + public List fetchBrapiTrialsByDbId(Set trialDbIds, Program program) throws InternalServerException, ApiException { + // Initialize the list of BrAPI trials + List brapiTrials = null; - return trial; - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); + // Retrieve the trials from the DAO based on the provided trial database IDs and program + brapiTrials = brAPITrialDAO.getTrialsByDbIds(trialDbIds, program); + + // Check if all requested trials were found + if (trialDbIds.size() != brapiTrials.size()) { + // Identify the missing trial database IDs + Set missingIds = new HashSet<>(trialDbIds); + missingIds.removeAll(brapiTrials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toSet())); + + // Throw an exception with the list of missing trial database IDs + throw new IllegalStateException("Trial not found for trial dbid(s): " + String.join(COMMA_DELIMITER, missingIds)); } + + // Return the list of retrieved BrAPI trials + return brapiTrials; } + + // TODO: also used in other workflow /** From 3ca1a30a4ddc76a172d14d2685e242ed4ada7d2b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 10 May 2024 12:05:09 -0400 Subject: [PATCH 43/61] Pass workflow id through to import services --- .../controllers/UploadController.java | 57 ++++++++++++++++++- .../model/imports/BrAPIImportService.java | 13 +---- .../model/imports/ImportServiceContext.java | 25 ++++++++ .../ExperimentImportService.java | 15 ++++- .../germplasm/GermplasmImportService.java | 11 +++- .../sample/SampleSubmissionImportService.java | 16 +++--- .../importer/services/FileImportService.java | 22 +++++-- 7 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index f9d55bd20..053f1dc3c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -114,7 +114,7 @@ public HttpResponse> commitData(@PathVariable UUID prog @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -140,7 +140,60 @@ public HttpResponse> previewData(@PathVariable UUID pro @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, null, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + Response response = new Response(result); + return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.notFound(); + } catch (AuthorizationException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.FORBIDDEN, e.getMessage()); + } catch (UnprocessableEntityException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (HttpStatusException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(e.getStatus(), e.getMessage()); + } + } + + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Produces(MediaType.APPLICATION_JSON) + @AddMetadata + @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, + @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @Body @Nullable Map userInput) { + try { + AuthenticatedUser actingUser = securityService.getUser(); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 1d520371c..d06454c30 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,18 +17,7 @@ package org.breedinginsight.brapps.importer.model.imports; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import java.util.List; public interface BrAPIImportService { String getImportTypeId(); @@ -48,6 +37,6 @@ default String getMissingUserInputMsg(String fieldName) { default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + ImportPreviewResponse process(ImportServiceContext context) throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java new file mode 100644 index 000000000..ef7639c2f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -0,0 +1,25 @@ +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; +import java.util.List; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportServiceContext { + private UUID workflowId; + private List brAPIImports; + private Table data; + private Program program; + private ImportUpload upload; + private User user; + private boolean commit; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index cd795564a..965a7d20f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -66,12 +67,22 @@ public String getMissingColumnMsg(String columnName) { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + // TODO: change to calling process directly on processor (not using processor manager and pass along workflowId) + if (context.getWorkflowId() != null) { + log.info("Workflow UUID: " + context.getWorkflowId()); + } + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index b4eac6b96..ce52f483f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -62,12 +63,18 @@ public String getImportTypeId() { } @Override - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); - response = processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + response = processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); return response; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 434626e68..2c8e8b7b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -59,13 +60,14 @@ public BrAPIImport getImportClass() { } @Override - public ImportPreviewResponse process(List brAPIImports, - Table data, - Program program, - ImportUpload upload, - User user, - Boolean commit) throws Exception { + public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); - return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + return processorManagerProvider.get().process(context.getBrAPIImports(), + processors, + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 950465459..960a8077f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -38,6 +38,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; @@ -328,7 +329,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -378,7 +379,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, AuthenticatedU } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -424,13 +425,22 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(List finalBrAPIImportList, Table data, Program program, - ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, - AuthenticatedUser actingUser) { + private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, + AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - importService.process(finalBrAPIImportList, data, program, upload, user, commit); + ImportServiceContext context = ImportServiceContext.builder() + .workflowId(workflowId) + .brAPIImports(finalBrAPIImportList) + .data(data) + .program(program) + .upload(upload) + .user(user) + .commit(commit) + .build(); + importService.process(context); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); From 7108089af495dc99ad3e983b6c4cfb89f777d54d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 10 May 2024 14:58:33 -0400 Subject: [PATCH 44/61] make link method static --- .../AppendOverwritePhenotypesWorkflow.java | 3 +- .../middleware/process/DataValidation.java | 65 +++++++++++++++++++ .../middleware/process/FieldValidation.java | 65 +++++++++++++++++++ .../process/ImportPreviewStatistics.java | 36 ++++++++++ .../process/NewPendingBrAPIObjects.java | 65 +++++++++++++++++++ .../middleware/process/TraitVerification.java | 65 +++++++++++++++++++ .../read/brapi/RequiredBrAPIData.java | 13 ++-- .../read/brapi/RequiredObservationUnits.java | 1 - .../experiment/middleware/Middleware.java | 2 +- 9 files changed, 306 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index b97e40536..2d8664968 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -5,6 +5,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.HandleErr; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; +import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; @@ -26,7 +27,7 @@ public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { - this.middleware.link(handleErrProvider.get(), + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(handleErrProvider.get(), validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java new file mode 100644 index 000000000..65b4c7998 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class DataValidation extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public DataValidation(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java new file mode 100644 index 000000000..d27190547 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class FieldValidation extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public FieldValidation(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java new file mode 100644 index 000000000..a2a446f4c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -0,0 +1,36 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; + +import javax.inject.Inject; +import javax.inject.Provider; + +@Slf4j +public class ImportPreviewStatistics extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + private Provider traitVerificationProvider; + private Provider newPendingBrAPIObjectsProvider; + private Provider dataValidationProvider; + private Provider fieldValidationProvider; + + @Inject + public ImportPreviewStatistics(Provider traitVerificationProvider, + Provider newPendingBrAPIObjectsProvider, + Provider dataValidationProvider, + Provider fieldValidationProvider) { + + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + traitVerificationProvider.get(), // Verify observation variable columns in import belong to program + newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data + dataValidationProvider.get(), // Validate data + fieldValidationProvider.get()); // Validate fields + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("generating import preview statistics"); + return this.middleware.process(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java new file mode 100644 index 000000000..fc6b63f98 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class NewPendingBrAPIObjects extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public NewPendingBrAPIObjects(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java new file mode 100644 index 000000000..facbe014f --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -0,0 +1,65 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class TraitVerification extends ExpUnitMiddleware { + ObservationUnitService observationUnitService; + + @Inject + public TraitVerification(ObservationUnitService observationUnitService) { + this.observationUnitService = observationUnitService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + Program program; + Set expUnitIds; + List missingIds; + List brapiUnits; + List> pendingUnits; + Map> pendingUnitById; + Map> pendingUnitByNameNoScope; + + log.debug("fetching required exp units from BrAPI service"); + program = context.getImportContext().getProgram(); + + // Collect deltabreed-generated exp unit ids listed in the import + expUnitIds = context.getExpUnitContext().getReferenceOUIds(); + try { + // For each id fetch the observation unit from the brapi data store + brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + + // Construct pending import objects from the units + pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + + // Construct a hashmap to look up the pending unit by ID + pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + + // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed + pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + + // add maps to the context for use in processing import + context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); + context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); + } catch (ApiException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java index 2446c9358..180e6957a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredBrAPIData.java @@ -25,12 +25,13 @@ public RequiredBrAPIData(Provider requiredObservationU Provider requiredDatasetsProvider, Provider requiredGermplasmProvider) { - this.middleware.link(requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids - requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units - requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units - requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units - requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units - requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( + requiredObservationUnitsProvider.get(), // Fetch the BrAPI units for the required exp unit ids + requiredTrialsProvider.get(), // Fetch the BrAPI trials belonging to the exp units + requiredStudiesProvider.get(), // Fetch the BrAPI studies belonging to the exp units + requiredLocationsProvider.get(), // Fetch the BrAPI locations belonging to the exp units + requiredDatasetsProvider.get(), // Fetch the dataset belonging to the exp units + requiredGermplasmProvider.get()); // Fetch the germplasm belonging to the exp units } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java index 5a3fa8301..ba19b96f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredObservationUnits.java @@ -5,7 +5,6 @@ import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java index 108d6cd88..c3c78cdd6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/middleware/Middleware.java @@ -10,7 +10,7 @@ public abstract class Middleware { /** * Builds chains of middleware objects. */ - public Middleware link(Middleware first, Middleware... chain) { + public static Middleware link(Middleware first, Middleware... chain) { Middleware head = first; for (Middleware nextInChain: chain) { nextInChain.prior = head.getLastLink(); From c8e0bfe736957eb5b3ddd5186cd7ab6096580c4a Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 16:45:25 -0400 Subject: [PATCH 45/61] Add workflow factory to create workflows from ids --- .../daos/ImportMappingWorkflowDAO.java | 12 +++ .../model/imports/ImportServiceContext.java | 21 ++++- .../ExperimentImportService.java | 8 +- .../model/workflow/ImportContext.java | 47 +++++++++++ .../model/workflow/ImportMappingWorkflow.java | 2 - .../model/workflow/ProcessedData.java | 31 +++++++ .../importer/model/workflow/Workflow.java | 36 ++++++++ .../importer/services/FileImportService.java | 14 +++- .../workflow/CreateNewExperimentWorkflow.java | 50 +++++++++++ .../services/workflow/WorkflowFactory.java | 84 +++++++++++++++++++ 10 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index c2ad58c23..21aca395a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -9,6 +9,7 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.UUID; import static org.breedinginsight.dao.db.Tables.*; @@ -36,4 +37,15 @@ public List getWorkflowsByImportMappingId(UUID mappingId) .fetch(ImportMappingWorkflow::parseSQLRecord); } + /** + * Retrieves a workflow by its ID. + * + * @param workflowId The ID of the workflow to retrieve. + * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. + */ + public Optional getWorkflowById(UUID workflowId) { + return Optional.ofNullable(fetchOneById(workflowId)) + .map(ImportMappingWorkflow::new); + } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index ef7639c2f..45393f67c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -1,12 +1,29 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.model.imports; import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; import java.util.List; -import java.util.UUID; @Getter @Setter @@ -15,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private UUID workflowId; + private Workflow workflow; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 965a7d20f..c6f68b251 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -72,10 +72,12 @@ public ImportPreviewResponse process(ImportServiceContext context) ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); - // TODO: change to calling process directly on processor (not using processor manager and pass along workflowId) - if (context.getWorkflowId() != null) { - log.info("Workflow UUID: " + context.getWorkflowId()); + + if (context.getWorkflow() != null) { + log.info("Workflow: " + context.getWorkflow().getName()); } + + // TODO: change to calling workflow process instead of processor manager response = processorManagerProvider.get().process(context.getBrAPIImports(), processors, context.getData(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java new file mode 100644 index 000000000..121ef6b19 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -0,0 +1,47 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import tech.tablesaw.api.Table; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +@NoArgsConstructor +public class ImportContext { + private UUID workflowId; + private ImportUpload upload; + private List importRows; + private Map mappedBrAPIImport; + private Table data; + private Program program; + private User user; + private boolean commit; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 51a70bb5a..0a23e1aae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -36,12 +36,10 @@ @NoArgsConstructor @SuperBuilder() public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); } - public static ImportMappingWorkflow parseSQLRecord(Record record) { return ImportMappingWorkflow.builder() diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java new file mode 100644 index 000000000..e73d6ad48 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -0,0 +1,31 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.Map; + +@Data +@Builder +@ToString +@NoArgsConstructor +public class ProcessedData { + private Map statistics; +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..9ea0e7bf9 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.workflow; + +public interface Workflow { + + /** + * Processes the given import context and returns the processed data. + * + * @param context the import context containing the necessary data for processing + * @return the processed data + */ + ProcessedData process(ImportContext context); + + /** + * Retrieves the name of the Workflow for logging display purposes. + * + * @return the name of the Workflow + */ + String getName(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 960a8077f..2c3f7854f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -44,6 +44,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; @@ -86,12 +88,13 @@ public class FileImportService { private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, UserService userService) { + ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -104,6 +107,7 @@ public class FileImportService { this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -431,8 +435,14 @@ private void processFile(UUID workflowId, List finalBrAPIImportList // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { + Workflow workflow = null; + if (workflowId != null) { + Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); + workflow = optionalWorkflow.orElse(null); + } + ImportServiceContext context = ImportServiceContext.builder() - .workflowId(workflowId) + .workflow(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java new file mode 100644 index 000000000..6f6ab661a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column + * value in the import_mapping_workflow db table + */ +@Prototype +@Named("CreateNewExperimentWorkflow") +public class CreateNewExperimentWorkflow implements Workflow { + + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewExperimentWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java new file mode 100644 index 000000000..60a2fd884 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -0,0 +1,84 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.workflow; + +import io.micronaut.context.ApplicationContext; +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.exceptions.NoSuchBeanException; +import io.micronaut.inject.qualifiers.Qualifiers; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; +import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.UUID; + +@Factory +@Slf4j +public class WorkflowFactory { + + private final ImportMappingWorkflowDAO importMappingWorkflowDAO; + private final ApplicationContext applicationContext; + + @Inject + public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, + ApplicationContext applicationContext) { + this.importMappingWorkflowDAO = importMappingWorkflowDAO; + this.applicationContext = applicationContext; + } + + /** + * Produces the appropriate workflow instance based on the import context + * + * @param context the import context + * @return an Optional containing the workflow if found, otherwise an empty Optional + * + * @throws IllegalStateException + * @throws NoSuchBeanException + */ + public Optional getWorkflow(UUID workflowId) { + + if (workflowId != null) { + // construct workflow from db record + Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + if (workflowOptional.isPresent()) { + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); + + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; + } + + return Optional.of(workflow); + } + } + + return Optional.empty(); + } +} From 4391479ee6e2d257e0713c6c6b7cbfc631ee030c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 16:50:45 -0400 Subject: [PATCH 46/61] Add missing file header --- .../daos/ImportMappingWorkflowDAO.java | 19 +++++++++++++++++-- .../importer/services/FileImportService.java | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index 21aca395a..216506b5c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -1,13 +1,28 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.brapps.importer.daos; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; import org.jooq.Configuration; import org.jooq.DSLContext; -import org.jooq.Record; import javax.inject.Inject; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 2c3f7854f..ef6c5f884 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -440,7 +440,7 @@ private void processFile(UUID workflowId, List finalBrAPIImportList Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); workflow = optionalWorkflow.orElse(null); } - + ImportServiceContext context = ImportServiceContext.builder() .workflow(workflow) .brAPIImports(finalBrAPIImportList) From 87ffdb53d134b43b0955aacee063c351d29d5cfd Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 13 May 2024 17:08:39 -0400 Subject: [PATCH 47/61] Add bean to import mapping workflow model --- .../brapps/importer/model/workflow/ImportMappingWorkflow.java | 4 ++++ .../brapps/importer/model/workflow/ProcessedData.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 0a23e1aae..4ea2a888b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -36,15 +36,19 @@ @NoArgsConstructor @SuperBuilder() public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { + + public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); + this.setBean(importMappingWorkflowEntity.getBean()); } public static ImportMappingWorkflow parseSQLRecord(Record record) { return ImportMappingWorkflow.builder() .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) + .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) .build(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index e73d6ad48..f9f8196c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -23,7 +23,6 @@ import java.util.Map; @Data -@Builder @ToString @NoArgsConstructor public class ProcessedData { From 19e3f17a90eac5883644c3278df4ef4beff93c78 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 13 May 2024 18:24:18 -0400 Subject: [PATCH 48/61] validate dynamic columns --- .../model/v1/response/ValidationErrors.java | 2 +- .../middleware/process/TraitVerification.java | 97 ++++-- .../appendoverwrite/model/ExpUnitContext.java | 2 + .../experiment/create/model/PendingData.java | 2 + .../model/ExpImportProcessConstants.java | 4 +- .../service/ObservationVariableService.java | 288 ++++++++++++++++++ 6 files changed, 361 insertions(+), 34 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java diff --git a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java index 3c7f1d706..923529505 100644 --- a/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java +++ b/src/main/java/org/breedinginsight/api/model/v1/response/ValidationErrors.java @@ -17,6 +17,7 @@ package org.breedinginsight.api.model.v1.response; +import io.micronaut.http.HttpStatus; import lombok.*; import lombok.experimental.Accessors; @@ -46,7 +47,6 @@ public void addError(Integer rowNumber, ValidationError validationError){ newRow.addError(validationError); rowErrors.add(newRow); } - public void merge(ValidationErrors validationErrors){ for (RowValidationErrors rowValidationErrors: validationErrors.getRowErrors()){ for (ValidationError validationError: rowValidationErrors.getErrors()) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index facbe014f..5801f47d6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -1,60 +1,93 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import javax.inject.Inject; import java.util.*; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_REGEX; + @Slf4j public class TraitVerification extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; + ObservationVariableService observationVariableService; + FileMappingUtil fileMappingUtil; @Inject - public TraitVerification(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; + public TraitVerification(ObservationVariableService observationVariableService, + FileMappingUtil fileMappingUtil) { + this.observationVariableService = observationVariableService; + this.fileMappingUtil = fileMappingUtil; } @Override public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); + log.debug("verifying traits listed in import"); + + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = context.getImportContext().getImportRows(); + ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); + List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); + for (int rowNum = 0; rowNum < importRows.size(); i++) { + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + this.compensate(context, new MiddlewareError(() -> { + // any handling... + })); + } - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> colByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + // Add the map to the context for use in processing import + context.getPendingData().setTimeStampColByPheno(colByPheno); - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); + try { + // Fetch the traits named in the observation variable columns + Program program = context.getImportContext().getProgram(); + List traits = observationVariableService.fetchTraitsByName(varNames, program); - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { + // Sort the traits to match the order of the headers in the import file + List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + } catch (DoesNotExistException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java index 50ca1f4d2..6988cbf89 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/model/ExpUnitContext.java @@ -7,8 +7,10 @@ import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; import java.util.HashMap; import java.util.HashSet; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 340a39fba..58f97f727 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -9,6 +9,7 @@ import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.ProgramLocation; +import tech.tablesaw.columns.Column; import java.util.HashMap; import java.util.Map; @@ -26,5 +27,6 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + private Map> timeStampColByPheno; private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index 4da4414a1..a81a5067e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -5,10 +5,12 @@ public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; + public static final String TIMESTAMP_PREFIX = "TS:"; + public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; public enum ErrMessage { - MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); + private String value; ErrMessage(String value) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java new file mode 100644 index 000000000..bd4ced017 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -0,0 +1,288 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.service; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.columns.Column; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +public class ObservationVariableService { + private final OntologyService ontologyService; + + @Inject + public ObservationVariableService(OntologyService ontologyService) { + this.ontologyService = ontologyService; + } + + /** + * Fetches traits by name for the given set of variable names and program. + * + * This method fetches all stored traits for the specified program and filters them based on the set of variable names provided. + * It ensures that all requested observation variables are present and returns a list of matching traits. + * If any observation variables are missing, it throws an IllegalStateException with the missing variable names. + * + * @param varNames a set of variable names to fetch traits for + * @param program the program for which traits are fetched + * @return a list of traits filtered by the provided variable names + * @throws DoesNotExistException if the program or traits do not exist + * @throws IllegalStateException if any requested observation variables are missing + */ + public List fetchTraitsByName(Set varNames, Program program) throws DoesNotExistException, IllegalStateException { + List traits = null; + + // Fetch all stored traits for the program + List programTraits = ontologyService.getTraitsByProgramId(program.getId(), true); + + // Only keep traits that are in the set of names + List upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); + traits = programTraits.stream().filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())).collect(Collectors.toList()); + + // If any requested observation variables are missing, throw an IllegalStateException + if (varNames.size() != traits.size()) { + Set missingVarNames = new HashSet<>(varNames); + missingVarNames.removeAll(traits.stream().map(TraitEntity::getObservationVariableName).collect(Collectors.toSet())); + throw new IllegalStateException("Observation variables not found for name(s): " + String.join(ExpImportProcessConstants.COMMA_DELIMITER, missingVarNames)); + } + + return traits; + } + + public Optional> validateMatchedTimestamps(Set observationVariableNames, + List> timestampCols) { + Optional> ve = Optional.empty(); + // Check that each ts column corresponds to a phenotype column + List valErrs = timestampCols.stream() + .filter(col -> !(observationVariableNames.contains(col.name().replaceFirst(ExpImportProcessConstants.TIMESTAMP_REGEX, StringUtils.EMPTY)))) + .map(col -> new ValidationError(col.name().toString(), String.format("Timestamp column %s lacks corresponding phenotype column", col.name().toString()), HttpStatus.UNPROCESSABLE_ENTITY)) + .collect(Collectors.toList()); + if (valErrs.size() > 0) { + ve = Optional.of(valErrs); + } + + return ve; + } + // TODO: used with expUnit workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + ExpUnitContext expUnitContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + validateObservation(importHash); + } + } + + // TODO: used with create workflow + public void validateObservations(PendingData pendingData, + int rowNum, + ImportContext importContext, + List> phenotypeCols, + CaseInsensitiveMap colVarMap) { + + + for (Column phenoCol : phenotypeCols) { + String importHash; + String importObsValue = phenoCol.getString(rowNum); + + importHash = getImportObservationHash(importRow, phenoCol.name()); + + validateObservation(importHash); + } + + } + + // TODO: used by both workflows + private void validateObservation(String importHash) { + + + String importObsValue = phenoCol.getString(rowNum); + + + // error if import observation data already exists and user has not selected to overwrite + if (commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + addRowError( + phenoCol.name(), + String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), + validationErrors, rowNum + ); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, + reason, + user.getId(), + timestamp + ); + + // create the changelog field in additional info if it does not already exist + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // add a new entry to the changelog + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); + } + + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if (isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); + existingObs.setObservationVariableName(phenoCol.name()); + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if (!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + } else { + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + + //Timestamp validation + if (timeStampColByPheno.containsKey(phenoCol.name())) { + Column timeStampCol = timeStampColByPheno.get(phenoCol.name()); + validateTimeStampValue(timeStampCol.getString(rowNum), timeStampCol.name(), validationErrors, rowNum); + } + } + + } + + // TODO: used by both workflows + private void updateObservationDependencyValues(Program program) { + String programKey = program.getKey(); + + // update the observations study DbIds, Observation Unit DbIds and Germplasm DbIds + this.observationUnitByNameNoScope.values().stream() + .map(PendingImportObject::getBrAPIObject) + .forEach(obsUnit -> updateObservationDbIds(obsUnit, programKey)); + + // Update ObservationVariable DbIds + List traits = getTraitList(program); + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } + for (PendingImportObject observation : this.observationByHash.values()) { + String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); + if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { + String observationVariableDbId = traitMap.get(observationVariableName).getObservationVariableDbId(); + observation.getBrAPIObject().setObservationVariableDbId(observationVariableDbId); + } + } + } + + // TODO: used by both workflows + public List commitNewPendingObservationsToBrAPIStore(ImportContext context, PendingData pendingData) { + // filter out observations with no 'value' so they will not be saved + List newObservations = ProcessorData.getNewObjects(this.observationByHash) + .stream() + .filter(obs -> !obs.getValue().isBlank()) + .collect(Collectors.toList()); + + updateObservationDependencyValues(program); + return brAPIObservationDAO.createBrAPIObservations(newObservations, program.getId(), upload); + + } + + // TODO: used by both workflows + public List commitUpdatedPendingObservationsToBrAPIStore(ImportContext importContext, PendingData pendingData) { + List updatedObservations = new ArrayList<>(); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); + + for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { + String id = entry.getKey(); + BrAPIObservation observation = entry.getValue(); + try { + if (observation == null) { + throw new Exception("Null observation"); + } + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + updatedObservations.add(updatedObs); + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { + String message; + if (!Objects.equals(observation.getValue(), updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } + + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + } + + return updatedObservations; + } +} From 964c2f3e9b707dfd330f24dceb0349da5f010077 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 14 May 2024 09:57:25 -0400 Subject: [PATCH 49/61] hash observations and add map to context --- .../middleware/process/TraitVerification.java | 45 +++++++++++++-- .../experiment/create/model/PendingData.java | 2 + .../service/ObservationService.java | 56 +++++++++++++++++-- .../experiment/service/StudyService.java | 1 + 4 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index 5801f47d6..2c0c3a913 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -1,27 +1,31 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; -import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; -import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.api.model.v1.response.ValidationErrors; import javax.inject.Inject; import java.util.*; @@ -32,12 +36,18 @@ @Slf4j public class TraitVerification extends ExpUnitMiddleware { ObservationVariableService observationVariableService; + ObservationService observationService; + BrAPIObservationDAO brAPIObservationDAO; FileMappingUtil fileMappingUtil; @Inject public TraitVerification(ObservationVariableService observationVariableService, + BrAPIObservationDAO brAPIObservationDAO, + ObservationService observationService, FileMappingUtil fileMappingUtil) { this.observationVariableService = observationVariableService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.observationService = observationService; this.fileMappingUtil = fileMappingUtil; } @@ -87,7 +97,30 @@ public boolean process(ExpUnitMiddlewareContext context) { // Sort the traits to match the order of the headers in the import file List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); - } catch (DoesNotExistException e) { + + // Read any observation data stored for these traits + Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + + // Construct helper lookup tables to use for hashing observations + Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); + Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + // Hash observations using a signature of unit, variable, and study names + Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ + return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + variableNameByDbId.get(o.getObservationVariableDbId()), + studyNameByDbId.get(o.getStudyDbId())); + }, o->o)); + + // Add the observation map to the context for use in processing import + context.getPendingData().setExistingObsByObsHash(observationByObsHash); + } catch (DoesNotExistException | ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 58f97f727..7799e9574 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -5,6 +5,7 @@ import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -28,5 +29,6 @@ public class PendingData { private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; private Map> timeStampColByPheno; + private Map existingObsByObsHash; private ValidationErrors validationErrors; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index f3ac20000..e421fa54d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -3,9 +3,11 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.http.server.exceptions.InternalServerException; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; @@ -22,13 +24,59 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; public class ObservationService { + public String getObservationHash(String observationUnitName, String variableName, String studyName) { + String concat = DigestUtils.sha256Hex(observationUnitName) + + DigestUtils.sha256Hex(variableName) + + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); + return DigestUtils.sha256Hex(concat); + } + public Map fetchExistingObservations(List referencedTraits, Program program) throws ApiException { + Set ouDbIds = new HashSet<>(); + Set variableDbIds = new HashSet<>(); + Map variableNameByDbId = new HashMap<>(); + Map ouNameByDbId = new HashMap<>(); + Map studyNameByDbId = studyByNameNoScope.values() + .stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + studyNameByDbId.keySet().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); + } + }); + + for (Trait referencedTrait : referencedTraits) { + variableDbIds.add(referencedTrait.getObservationVariableDbId()); + variableNameByDbId.put(referencedTrait.getObservationVariableDbId(), referencedTrait.getObservationVariableName()); + } + + List existingObservations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, variableDbIds, program); + + return existingObservations.stream() + .map(obs -> { + String studyName = studyNameByDbId.get(obs.getStudyDbId()); + String variableName = variableNameByDbId.get(obs.getObservationVariableDbId()); + String ouName = ouNameByDbId.get(obs.getObservationUnitDbId()); + + String key = getObservationHash(createObservationUnitKey(studyName, ouName), variableName, studyName); + + return Map.entry(key, obs); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } // TODO: used with expUnit workflow public void validateObservations(PendingData pendingData, int rowNum, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index baf646689..53015159b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -52,6 +52,7 @@ public StudyService(BrAPISeasonDAO brAPISeasonDAO, this.brAPIStudyDAO = brAPIStudyDAO; } + // TODO: used by both workflows public PendingImportObject processAndCacheStudy( BrAPIStudy existingStudy, From 186ee1d893d47ccb2d3290e6374df106d52f4cdd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 14 May 2024 10:00:48 -0400 Subject: [PATCH 50/61] add logging statement --- .../appendoverwrite/middleware/process/TraitVerification.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index 2c0c3a913..59936a333 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -99,6 +99,7 @@ public boolean process(ExpUnitMiddlewareContext context) { List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); // Read any observation data stored for these traits + log.debug("fetching observation data stored for traits"); Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); From dc2ea96c6879ce858f8f817f4bcc5f932674df5b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 14 May 2024 10:08:06 -0400 Subject: [PATCH 51/61] fix for loop --- .../appendoverwrite/middleware/process/TraitVerification.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java index 59936a333..a2a04194d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java @@ -73,7 +73,8 @@ public boolean process(ExpUnitMiddlewareContext context) { List importRows = context.getImportContext().getImportRows(); ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); - for (int rowNum = 0; rowNum < importRows.size(); i++) { + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); } From 1c1df39b03dd513adff46bc8bd7e8d9dde19bff7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 10:29:04 -0400 Subject: [PATCH 52/61] Added position column to workflows for explicit ordering --- .../daos/ImportMappingWorkflowDAO.java | 4 +++- .../model/workflow/ImportMappingWorkflow.java | 2 ++ .../services/workflow/WorkflowFactory.java | 4 +++- .../V1.22.0__add_experiment_workflows.sql | 19 ++++++++++++++----- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java index 216506b5c..1b60bda45 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java @@ -40,7 +40,8 @@ public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { } /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. + * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by + * position for proper ordering on the front end. * * @param mappingId The UUID of the mapping to retrieve the workflows for. * @return A list of ImportMappingWorkflow objects. @@ -49,6 +50,7 @@ public List getWorkflowsByImportMappingId(UUID mappingId) return dsl.select() .from(IMPORTER_MAPPING_WORKFLOW) .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) + .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) .fetch(ImportMappingWorkflow::parseSQLRecord); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java index 4ea2a888b..8dc5800bc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java @@ -42,6 +42,7 @@ public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflow this.setId(importMappingWorkflowEntity.getId()); this.setName(importMappingWorkflowEntity.getName()); this.setBean(importMappingWorkflowEntity.getBean()); + this.setPosition(importMappingWorkflowEntity.getPosition()); } public static ImportMappingWorkflow parseSQLRecord(Record record) { @@ -49,6 +50,7 @@ public static ImportMappingWorkflow parseSQLRecord(Record record) { .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) + .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) .build(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java index 60a2fd884..d265b66a0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -48,7 +48,7 @@ public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, * Produces the appropriate workflow instance based on the import context * * @param context the import context - * @return an Optional containing the workflow if found, otherwise an empty Optional + * @return an Optional containing the workflow if id is not null, otherwise an empty Optional * * @throws IllegalStateException * @throws NoSuchBeanException @@ -58,6 +58,8 @@ public Optional getWorkflow(UUID workflowId) { if (workflowId != null) { // construct workflow from db record Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); + + // TODO: look at this optional handling if (workflowOptional.isPresent()) { ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { String msg = "Must have record in db for workflowId"; diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql index d5c4b9556..2c9d4d547 100644 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql @@ -15,12 +15,21 @@ * limitations under the License. */ +/** + Table maps workflows to import mappings and provides required configuration options + + mapping_id - link to importer_mapping that this provides a workflow for + name - name that will be displayed on front end + bean - must match @Named("") annotation on Workflow class + position - for ordering records explicitly, wanted for front end default option and order + */ CREATE TABLE importer_mapping_workflow ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, mapping_id UUID NOT NULL, name TEXT NOT NULL, - bean TEXT NOT NULL + bean TEXT NOT NULL, + position INTEGER NOT NULL ); ALTER TABLE importer_mapping_workflow @@ -33,10 +42,10 @@ DECLARE BEGIN exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean) +INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow'), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow'), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow'); + (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), + (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), + (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); END $$; \ No newline at end of file From 36944a510a2dade39bd1dff565621a1b2f6f6e41 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 10:45:35 -0400 Subject: [PATCH 53/61] Added additional workflow skeletons --- .../AppendOverwritePhenotypesWorkflow.java | 50 +++++++++++++++++++ .../CreateNewEnvironmentWorkflow.java | 50 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java new file mode 100644 index 000000000..21df19be6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate + * bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("AppendOverwritePhenotypesWorkflow") +public class AppendOverwritePhenotypesWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "AppendOverwritePhenotypesWorkflow"; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java new file mode 100644 index 000000000..5776ab59b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -0,0 +1,50 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; + +import javax.inject.Named; + +/** + * This class represents a workflow for adding new environments to an existing experiment. The bean name must match + * the appropriate bean column value in the import_mapping_workflow db table + */ + +@Prototype +@Named("CreateNewEnvironmentWorkflow") +public class CreateNewEnvironmentWorkflow implements Workflow { + @Override + public ProcessedData process(ImportContext context) { + // TODO + return null; + } + + /** + * Retrieves the name of the workflow. This is used for logging display purposes. + * + * @return the name of the workflow + */ + @Override + public String getName() { + return "CreateNewEnvironmentWorkflow"; + } +} From 7a9af10ed0356559c6a27de474c86697f0e38f58 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 14 May 2024 14:01:41 -0400 Subject: [PATCH 54/61] Clean up factory code --- .../services/workflow/WorkflowFactory.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java index d265b66a0..17d6fe64e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java @@ -59,26 +59,23 @@ public Optional getWorkflow(UUID workflowId) { // construct workflow from db record Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - // TODO: look at this optional handling - if (workflowOptional.isPresent()) { - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); + ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { + String msg = "Must have record in db for workflowId"; + log.error(msg); + return new IllegalStateException(msg); + }); - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); + // newer versions of micronaut have fancier ways to do this using annotations with provider but as + // far as I can tell it's not available in 2.5 + Workflow workflow; + try { + workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); + } catch (NoSuchBeanException e) { + log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); + throw e; } + + return Optional.of(workflow); } return Optional.empty(); From 027855f3bf43618ce0f54b811387247d0cb66057 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 15 May 2024 15:19:08 -0400 Subject: [PATCH 55/61] construct pending observations --- .../experiment/ExperimentUtilities.java | 15 +- .../process/ImportPreviewStatistics.java | 7 +- .../process/NewPendingBrAPIObjects.java | 65 ----- .../middleware/process/TraitVerification.java | 133 --------- .../process/brapi/NewPendingBrAPIObjects.java | 49 ++++ .../process/brapi/PendingObservation.java | 254 ++++++++++++++++++ .../read/brapi/RequiredDatasets.java | 25 +- .../experiment/create/model/PendingData.java | 1 + .../model/ExpImportProcessConstants.java | 5 + .../experiment/service/DatasetService.java | 8 +- .../service/ObservationService.java | 20 ++ .../experiment/service/StudyService.java | 5 +- 12 files changed, 360 insertions(+), 227 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java index d22f9ae2f..e321c2f59 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java @@ -12,10 +12,9 @@ import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.*; import java.util.stream.Collectors; public class ExperimentUtilities { @@ -52,6 +51,14 @@ public V getSingleEntryValue(Map map, String message) throws Unproc return map.values().iterator().next(); } + public static Optional getSingleEntryValue(Map map) { + Optional value = Optional.empty(); + if (map.size() == 1) { + value = Optional.ofNullable(map.values().iterator().next()); + } + return value; + } + /* * this will add the given year to the additionalInfo field of the BrAPIStudy (if it does not already exist) * */ diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index a2a446f4c..0ead6a47e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -2,6 +2,8 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.NewPendingBrAPIObjects; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.PendingObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; @@ -10,19 +12,16 @@ @Slf4j public class ImportPreviewStatistics extends ExpUnitMiddleware { ExpUnitMiddleware middleware; - private Provider traitVerificationProvider; private Provider newPendingBrAPIObjectsProvider; private Provider dataValidationProvider; private Provider fieldValidationProvider; @Inject - public ImportPreviewStatistics(Provider traitVerificationProvider, - Provider newPendingBrAPIObjectsProvider, + public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, Provider dataValidationProvider, Provider fieldValidationProvider) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( - traitVerificationProvider.get(), // Verify observation variable columns in import belong to program newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data dataValidationProvider.get(), // Validate data fieldValidationProvider.get()); // Validate fields diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java deleted file mode 100644 index fc6b63f98..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/NewPendingBrAPIObjects.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class NewPendingBrAPIObjects extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; - - @Inject - public NewPendingBrAPIObjects(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java deleted file mode 100644 index a2a04194d..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/TraitVerification.java +++ /dev/null @@ -1,133 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.FileMappingUtil; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; -import org.breedinginsight.dao.db.tables.pojos.TraitEntity; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.Trait; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.Utilities; -import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_REGEX; - -@Slf4j -public class TraitVerification extends ExpUnitMiddleware { - ObservationVariableService observationVariableService; - ObservationService observationService; - BrAPIObservationDAO brAPIObservationDAO; - FileMappingUtil fileMappingUtil; - - @Inject - public TraitVerification(ObservationVariableService observationVariableService, - BrAPIObservationDAO brAPIObservationDAO, - ObservationService observationService, - FileMappingUtil fileMappingUtil) { - this.observationVariableService = observationVariableService; - this.brAPIObservationDAO = brAPIObservationDAO; - this.observationService = observationService; - this.fileMappingUtil = fileMappingUtil; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - log.debug("verifying traits listed in import"); - - // Get all the dynamic columns of the import - ImportUpload upload = context.getImportContext().getUpload(); - Table data = context.getImportContext().getData(); - String[] dynamicColNames = upload.getDynamicColumnNames(); - List> dynamicCols = data.columns(dynamicColNames); - - // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); - - // Collect the columns for observation timestamps - List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); - - // Construct validation errors for any timestamp columns that don't have a matching variable column - List importRows = context.getImportContext().getImportRows(); - ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); - List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); - for (int i = 0; i < importRows.size(); i++) { - int rowNum = i; - tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); - } - - // Stop processing the import if there are unmatched timestamp columns - if (tsValErrs.size() > 0) { - this.compensate(context, new MiddlewareError(() -> { - // any handling... - })); - } - - //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval - Map> colByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); - - // Add the map to the context for use in processing import - context.getPendingData().setTimeStampColByPheno(colByPheno); - - try { - // Fetch the traits named in the observation variable columns - Program program = context.getImportContext().getProgram(); - List traits = observationVariableService.fetchTraitsByName(varNames, program); - - // Sort the traits to match the order of the headers in the import file - List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); - - // Read any observation data stored for these traits - log.debug("fetching observation data stored for traits"); - Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); - Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); - List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); - - // Construct helper lookup tables to use for hashing observations - Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); - Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); - Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() - .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) - .map(PendingImportObject::getBrAPIObject) - .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - - // Hash observations using a signature of unit, variable, and study names - Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ - return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), - variableNameByDbId.get(o.getObservationVariableDbId()), - studyNameByDbId.get(o.getStudyDbId())); - }, o->o)); - - // Add the observation map to the context for use in processing import - context.getPendingData().setExistingObsByObsHash(observationByObsHash); - } catch (DoesNotExistException | ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java new file mode 100644 index 000000000..8f7e6362a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/NewPendingBrAPIObjects.java @@ -0,0 +1,49 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; + +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import javax.inject.Provider; +import java.math.BigInteger; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +public class NewPendingBrAPIObjects extends ExpUnitMiddleware { + ExpUnitMiddleware middleware; + Provider pendingObservationProvider; + @Inject + public NewPendingBrAPIObjects(Provider pendingObservationProvider) { + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(pendingObservationProvider.get()); // Construct new pending observation + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("constructing new pending BrAPI objects"); + + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java new file mode 100644 index 000000000..162ee0842 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -0,0 +1,254 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; + +import com.google.gson.Gson; +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationVariableService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.utilities.Utilities; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.ErrMessage.MULTIPLE_EXP_TITLES; + +@Slf4j +public class PendingObservation extends ExpUnitMiddleware { + StudyService studyService; + ObservationVariableService observationVariableService; + ObservationService observationService; + BrAPIObservationDAO brAPIObservationDAO; + FileMappingUtil fileMappingUtil; + Gson gson; + + @Inject + public PendingObservation(StudyService studyService, + ObservationVariableService observationVariableService, + BrAPIObservationDAO brAPIObservationDAO, + ObservationService observationService, + FileMappingUtil fileMappingUtil, + Gson gson) { + this.studyService = studyService; + this.observationVariableService = observationVariableService; + this.brAPIObservationDAO = brAPIObservationDAO; + this.observationService = observationService; + this.fileMappingUtil = fileMappingUtil; + this.gson = gson; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + log.debug("verifying traits listed in import"); + + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Collect the columns for observation timestamps + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); + + // Construct validation errors for any timestamp columns that don't have a matching variable column + List importRows = context.getImportContext().getImportRows(); + ValidationErrors validationErrors = context.getPendingData().getValidationErrors(); + List tsValErrs = observationVariableService.validateMatchedTimestamps(varNames, timestampCols).orElse(new ArrayList<>()); + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i; + tsValErrs.forEach(validationError -> validationErrors.addError(rowNum, validationError)); + } + + // Stop processing the import if there are unmatched timestamp columns + if (tsValErrs.size() > 0) { + this.compensate(context, new MiddlewareError(() -> { + // any handling... + })); + } + + //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval + Map> tsColByPheno = timestampCols.stream().collect(Collectors.toMap(col -> col.name().replaceFirst(TIMESTAMP_REGEX, StringUtils.EMPTY), col -> col)); + + // Add the map to the context for use in processing import + context.getPendingData().setTimeStampColByPheno(tsColByPheno); + + try { + // Fetch the traits named in the observation variable columns + Program program = context.getImportContext().getProgram(); + List traits = observationVariableService.fetchTraitsByName(varNames, program); + + // Sort the traits to match the order of the headers in the import file + List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); + + // Get the pending observation dataset + PendingImportObject pendingTrial = ExperimentUtilities.getSingleEntryValue(context.getPendingData().getTrialByNameNoScope()).orElseThrow(()->new UnprocessableEntityException(MULTIPLE_EXP_TITLES.getValue())); + String datasetName = String.format("Observation Dataset [%s-%s]", program.getKey(), pendingTrial.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXPERIMENT_NUMBER).getAsString()); + PendingImportObject pendingDataset = context.getPendingData().getObsVarDatasetByName().get(datasetName); + + // Add new phenotypes to the pending observation dataset list (NOTE: "obsVarName [programKey]" is used instead of obsVarDbId) + // TODO: Change to using actual dbIds as per the BrAPI spec, instead of namespaced obsVar names (necessary for Breedbase) + Set datasetObsVarDbIds = pendingDataset.getBrAPIObject().getData().stream().collect(Collectors.toSet()); + Set phenoDbIds = sortedTraits.stream().map(t->Utilities.appendProgramKey(t.getObservationVariableName(), program.getKey())).collect(Collectors.toSet()); + phenoDbIds.removeAll(datasetObsVarDbIds); + pendingDataset.getBrAPIObject().getData().addAll(phenoDbIds); + + // Update pending status + if (ImportObjectState.EXISTING == pendingDataset.getState()) { + pendingDataset.setState(ImportObjectState.MUTATED); + } + + // Read any observation data stored for these traits + log.debug("fetching observation data stored for traits"); + Set ouDbIds = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(u -> u.getBrAPIObject().getObservationUnitDbId()).collect(Collectors.toSet()); + Set varDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); + List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, varDbIds, program); + + // Construct helper lookup tables to use for hashing stored observation data + Map unitNameByDbId = context.getExpUnitContext().getPendingObsUnitByOUId().values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, BrAPIObservationUnit::getObservationUnitName)); + Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); + Map studyNameByDbId = context.getPendingData().getStudyByNameNoScope().values().stream() + .filter(pio -> StringUtils.isNotBlank(pio.getBrAPIObject().getStudyDbId())) + .map(PendingImportObject::getBrAPIObject) + .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); + + // Hash stored observation data using a signature of unit, variable, and study names + Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ + return observationService.getObservationHash(unitNameByDbId.get(o.getObservationUnitDbId()), + variableNameByDbId.get(o.getObservationVariableDbId()), + studyNameByDbId.get(o.getStudyDbId())); + }, o->o)); + + // Add the observation data map to the context for use in processing import + context.getPendingData().setExistingObsByObsHash(observationByObsHash); + + // Build new pending observation data for each phenotype + Map> pendingObservationByHash = new HashMap<>(); + for (Column column : phenotypeCols) { + + // Checking all import rows for data + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + String cellData = column.getString(rowNum); + + // Generate hash for looking up prior observation + String unitId = row.getExpUnitId(); + String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); + String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); + String phenoColumnName = column.name(); + String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); + + // Get timestamp if associated column + String timestamp = null; + if ( tsColByPheno.containsKey(phenoColumnName) ) { + timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + + // If timestamp is not valid, set to midnight + if (timestamp != null && !timestamp.isBlank() && (!observationService.validDateTimeValue(timestamp) || !observationService.validDateValue(timestamp))) { + timestamp += MIDNIGHT; + + // Add a validation error + ValidationError timestampValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Timestamp format is not valid for %s", tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum, timestampValErr); + } + } + + // Construct a pending observation for updates made to prior observations + if (observationByObsHash.containsKey(observationHash)) { + + // Clone the prior observation + BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); + + // Construct a pending observation + PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + + // Update the pending phenotypic data + if (!cellData.equals(observation.getValue())) { + pendingObservation.getBrAPIObject().setValue(cellData); + pendingObservation.setState(ImportObjectState.MUTATED); + } + + // Update the pending timestamp + if (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp())) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); + pendingObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + pendingObservation.setState(ImportObjectState.MUTATED); + } + + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingObservation); + + // Construct a pending observation for new data if no prior observation + } else { + UUID trialId = pendingTrial.getId(); + UUID studyId = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(); + String studyYear = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0); + String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); + + // Clone the observation unit + BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); + + // Generate a new ID for the observation + UUID observationID = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, context.getImportContext().isCommit(), program, context.getImportContext().getUser(), BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + + // Construct a pending observation + PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingObservation); + } + } + // Add the pending observation map to the context for use in processing the import + context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + } + } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { + this.compensate(context, new MiddlewareError(() -> { + throw new RuntimeException(e); + })); + } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java index 7e6967bc9..a2536ecf0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/read/brapi/RequiredDatasets.java @@ -49,27 +49,24 @@ public boolean process(ExpUnitMiddlewareContext context) { .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) .getAsString(); + try { + // Get the dataset belonging to required exp units + dataset = datasetService.fetchDatasetById(datasetId, program).orElseThrow(ApiException::new); + // Construct the pending dataset from the BrAPI observation variable list + pendingDataset = datasetService.constructPIOFromDataset(dataset, program); - // Get the dataset belonging to required exp units - try { - dataset = datasetService.fetchDatasetById(datasetId, program); + // Construct a hashmap to look up the pending dataset by dataset name + pendingDatasetByName = new HashMap<>(); + pendingDatasetByName.put(dataset.getListName(), pendingDataset); + + // Add the map to the context for use in processing import + context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); } catch (ApiException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); } - - // Construct the pending dataset from the BrAPI observation variable list - pendingDataset = datasetService.constructPIOFromDataset(dataset, program); - - // Construct a hashmap to look up the pending dataset by dataset name - pendingDatasetByName = new HashMap<>(); - pendingDatasetByName.put(dataset.getListName(), pendingDataset); - - // Add the map to the context for use in processing import - context.getPendingData().setObsVarDatasetByName(pendingDatasetByName); - return processNext(context); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java index 7799e9574..28eb14e29 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/PendingData.java @@ -28,6 +28,7 @@ public class PendingData { private Map> locationByName; private Map> obsVarDatasetByName; private Map> existingGermplasmByGID; + private Map> pendingObservationByHash; private Map> timeStampColByPheno; private Map existingObsByObsHash; private ValidationErrors validationErrors; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java index a81a5067e..0fc8b807c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/model/ExpImportProcessConstants.java @@ -1,13 +1,18 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.model; import com.fasterxml.jackson.annotation.JsonValue; +import io.micronaut.context.annotation.Property; public class ExpImportProcessConstants { public static final CharSequence COMMA_DELIMITER = ","; public static final String TIMESTAMP_PREFIX = "TS:"; public static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; + public static final String MIDNIGHT = "T00:00:00-00:00"; + @Property(name = "brapi.server.reference-source") + public static String BRAPI_REFERENCE_SOURCE; public enum ErrMessage { + MULTIPLE_EXP_TITLES("File contains more than one Experiment Title"), MISSING_OBS_UNIT_ID_ERROR("Experimental entities are missing ObsUnitIDs"), PREEXISTING_EXPERIMENT_TITLE("Experiment Title already exists"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 70169478e..6281e74d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -55,8 +55,8 @@ public DatasetService(BrAPIListDAO brapiListDAO) { * @return BrAPIListDetails object containing the details of the dataset * @throws ApiException if there is an issue with fetching the dataset details from the data source */ - public BrAPIListDetails fetchDatasetById(String id, Program program) throws ApiException { - BrAPIListDetails dataSetDetails = null; + public Optional fetchDatasetById(String id, Program program) throws ApiException { + Optional dataSetDetails = Optional.empty(); // Retrieve existing dataset summaries based on program ID and external reference List existingDatasets = brAPIListDAO @@ -71,9 +71,9 @@ public BrAPIListDetails fetchDatasetById(String id, Program program) throws ApiE } // Retrieve dataset details using the list DB ID from the existing dataset summary - dataSetDetails = brAPIListDAO + dataSetDetails = Optional.ofNullable(brAPIListDAO .getListById(existingDatasets.get(0).getListDbId(), program.getId()) - .getResult(); + .getResult()); return dataSetDetails; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index e421fa54d..67177dad5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -24,10 +24,30 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; public class ObservationService { + public boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + public boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } public String getObservationHash(String observationUnitName, String variableName, String studyName) { String concat = DigestUtils.sha256Hex(observationUnitName) + DigestUtils.sha256Hex(variableName) + diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 53015159b..7114e0b04 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -38,7 +38,7 @@ @Singleton @Slf4j public class StudyService { - + private final Map seasonDbIdToYearCache = new HashMap<>(); private final BrAPISeasonDAO brAPISeasonDAO; private final BrAPIStudyDAO brAPIStudyDAO; @@ -52,7 +52,6 @@ public StudyService(BrAPISeasonDAO brAPISeasonDAO, this.brAPIStudyDAO = brAPIStudyDAO; } - // TODO: used by both workflows public PendingImportObject processAndCacheStudy( BrAPIStudy existingStudy, @@ -111,7 +110,7 @@ public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brA } // TODO: used by both workflows - private String seasonDbIdToYear(String seasonDbId, UUID programId) { + public String seasonDbIdToYear(String seasonDbId, UUID programId) { String year = null; // TODO: add season objects to redis cache then just extract year from those // removing this for now here From d6ed109340eb7006043794b26778240cd961131a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 17 May 2024 14:20:04 -0400 Subject: [PATCH 56/61] cosntruct mapped pending import rows --- .../middleware/process/DataValidation.java | 65 -------------- .../process/ImportPreviewStatistics.java | 5 +- .../middleware/process/ValidationPrep.java | 84 +++++++++++++++++++ .../process/brapi/PendingObservation.java | 58 +++++++++---- .../experiment/service/DatasetService.java | 2 +- .../service/ObservationUnitService.java | 19 ----- .../experiment/service/StudyService.java | 2 +- .../experiment/service/TrialService.java | 5 +- 8 files changed, 132 insertions(+), 108 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java deleted file mode 100644 index 65b4c7998..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/DataValidation.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import org.breedinginsight.model.Program; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class DataValidation extends ExpUnitMiddleware { - ObservationUnitService observationUnitService; - - @Inject - public DataValidation(ObservationUnitService observationUnitService) { - this.observationUnitService = observationUnitService; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); - try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { - this.compensate(context, new MiddlewareError(() -> { - throw new RuntimeException(e); - })); - } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index 0ead6a47e..3f35afea5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -3,7 +3,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.NewPendingBrAPIObjects; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi.PendingObservation; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import javax.inject.Inject; @@ -13,12 +12,12 @@ public class ImportPreviewStatistics extends ExpUnitMiddleware { ExpUnitMiddleware middleware; private Provider newPendingBrAPIObjectsProvider; - private Provider dataValidationProvider; + private Provider dataValidationProvider; private Provider fieldValidationProvider; @Inject public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, - Provider dataValidationProvider, + Provider dataValidationProvider, Provider fieldValidationProvider) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java new file mode 100644 index 000000000..72fecfd21 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java @@ -0,0 +1,84 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; +import tech.tablesaw.api.Table; +import tech.tablesaw.columns.Column; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +import static org.breedinginsight.brapps.importer.services.processors.ObservationProcessor.getObservationHash; + +@Slf4j +public class ValidationPrep extends ExpUnitMiddleware { + ObservationService observationService; + + @Inject + public ValidationPrep(ObservationService observationService) { + this.observationService = observationService; + } + + @Override + public boolean process(ExpUnitMiddlewareContext context) { + // Get all the dynamic columns of the import + ImportUpload upload = context.getImportContext().getUpload(); + Table data = context.getImportContext().getData(); + String[] dynamicColNames = upload.getDynamicColumnNames(); + List> dynamicCols = data.columns(dynamicColNames); + + // Collect the columns for observation variable data + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); + + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Fetch the pending import for the row, creating one if null + PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); + + List> observations = mappedImportRow.getObservations(); + + String unitId = row.getObsUnitID(); + mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + String observationHash = observationService.getObservationHash( + pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(unitId).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() + ); + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + } + +// try { +// +// } catch (ApiException e) { +// this.compensate(context, new MiddlewareError(() -> { +// throw new RuntimeException(e); +// })); +// } + + return processNext(context); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index 162ee0842..e67ec1b04 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -16,6 +16,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; @@ -161,16 +162,27 @@ public boolean process(ExpUnitMiddlewareContext context) { // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); - for (Column column : phenotypeCols) { - // Checking all import rows for data - for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { - Integer rowNum = i; - ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Checking all import rows for data + for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { + Integer rowNum = i; + ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); + + // Construct the pending import for the row + PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); + String unitId = row.getObsUnitID(); + mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); + mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); + mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); + mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); + mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); + + // For each phenotype, construct the pending observations + for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); // Generate hash for looking up prior observation - String unitId = row.getExpUnitId(); String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); @@ -178,7 +190,7 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get timestamp if associated column String timestamp = null; - if ( tsColByPheno.containsKey(phenoColumnName) ) { + if (tsColByPheno.containsKey(phenoColumnName)) { timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); // If timestamp is not valid, set to midnight @@ -191,31 +203,34 @@ public boolean process(ExpUnitMiddlewareContext context) { } } - // Construct a pending observation for updates made to prior observations + // Construct a pending observation if this is a prior observation if (observationByObsHash.containsKey(observationHash)) { // Clone the prior observation BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Construct a pending observation - PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + PendingImportObject pendingPriorObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); // Update the pending phenotypic data if (!cellData.equals(observation.getValue())) { - pendingObservation.getBrAPIObject().setValue(cellData); - pendingObservation.setState(ImportObjectState.MUTATED); + pendingPriorObservation.getBrAPIObject().setValue(cellData); + pendingPriorObservation.setState(ImportObjectState.MUTATED); } // Update the pending timestamp if (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp())) { DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); - pendingObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - pendingObservation.setState(ImportObjectState.MUTATED); + pendingPriorObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + pendingPriorObservation.setState(ImportObjectState.MUTATED); } + // Set the pending prior observation in the pending import for the row + mappedImportRow.getObservations().add(pendingPriorObservation); + // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingObservation); + pendingObservationByHash.put(observationHash, pendingPriorObservation); // Construct a pending observation for new data if no prior observation } else { @@ -234,15 +249,22 @@ public boolean process(ExpUnitMiddlewareContext context) { BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, context.getImportContext().isCommit(), program, context.getImportContext().getUser(), BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); // Construct a pending observation - PendingImportObject pendingObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + PendingImportObject pendingNewObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + + // Set the new pending observation in the pending import for the row + mappedImportRow.getObservations().add(pendingNewObservation); // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingObservation); + pendingObservationByHash.put(observationHash, pendingNewObservation); } } - // Add the pending observation map to the context for use in processing the import - context.getPendingData().setPendingObservationByHash(pendingObservationByHash); + + // Set the pending import for the row + context.getImportContext().getMappedBrAPIImport().put(rowNum, mappedImportRow); } + + // Add the pending observation map to the context for use in processing the import + context.getPendingData().setPendingObservationByHash(pendingObservationByHash); } catch (DoesNotExistException | ApiException | UnprocessableEntityException e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java index 6281e74d3..399f23468 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/DatasetService.java @@ -93,7 +93,7 @@ public PendingImportObject constructPIOFromDataset(BrAPIListDe // Get the external reference for the dataset from the existing list BrAPIExternalReference xref = Utilities.getExternalReference(dataset.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) - .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId()); + .orElseThrow(() -> new IllegalStateException("External references weren't found for list (dbid): " + dataset.getListDbId())); // Create a PendingImportObject for the dataset with the existing list and reference ID return new PendingImportObject(ImportObjectState.EXISTING, dataset, UUID.fromString(xref.getReferenceId())); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index e19c196d1..dd8bc2856 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -351,25 +351,6 @@ public List commitUpdatedPendingObservationUnitToBrAPIStor return updatedUnits; } - private void updateObsUnitDependencyValues(String programKey) { - - // update study DbIds - this.studyByNameNoScope.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(study -> updateStudyDbId(study, programKey)); - - // update germplasm DbIds - this.existingGermplasmByGID.values() - .stream() - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); - } - private void updateStudyDbId(BrAPIStudy study, String programKey) { this.observationUnitByNameNoScope.values() .stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java index 7114e0b04..f6679e947 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/StudyService.java @@ -92,7 +92,7 @@ public PendingImportObject constructPIOFromBrapiStudy(BrAPIStudy brA // Retrieve external reference for the study BrAPIExternalReference xref = Utilities.getExternalReference(brAPIStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId()); + .orElseThrow(() -> new IllegalStateException("External references weren't found for study (dbid): " + brAPIStudy.getStudyDbId())); // Map season dbid to year String seasonDbId = brAPIStudy.getSeasons().get(0); // It is assumed that the study has only one season diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 430153d40..b82c5169d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -298,7 +298,7 @@ public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial tri String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId)); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId); return pio; } @@ -335,6 +335,9 @@ private Map> initializeTrialByNameNoScop return trialByName; } + private void initializeTrialsForExistingObservationUnits(Program program, Map> observationUnitByNameNoScope, Map> trialByName) { + } + // TODO: used by expunit workflow public Map> mapPendingTrialByOUId( String unitId, From 7cb1f589f9bd3693744154ea66eaa47edfc683af Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 17 May 2024 18:20:01 -0400 Subject: [PATCH 57/61] validate and record changelog --- .../middleware/process/FieldValidation.java | 28 +------ .../process/ImportPreviewStatistics.java | 3 - .../middleware/process/ValidationPrep.java | 84 ------------------- .../process/brapi/PendingObservation.java | 53 +++++++++++- 4 files changed, 52 insertions(+), 116 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java index d27190547..d28a59e2b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/FieldValidation.java @@ -25,36 +25,10 @@ public FieldValidation(ObservationUnitService observationUnitService) { @Override public boolean process(ExpUnitMiddlewareContext context) { - Program program; - Set expUnitIds; - List missingIds; - List brapiUnits; - List> pendingUnits; - Map> pendingUnitById; - Map> pendingUnitByNameNoScope; - log.debug("fetching required exp units from BrAPI service"); - program = context.getImportContext().getProgram(); - - // Collect deltabreed-generated exp unit ids listed in the import - expUnitIds = context.getExpUnitContext().getReferenceOUIds(); try { - // For each id fetch the observation unit from the brapi data store - brapiUnits = observationUnitService.getObservationUnitsByDbId(new HashSet<>(expUnitIds), program); - - // Construct pending import objects from the units - pendingUnits = brapiUnits.stream().map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - - // Construct a hashmap to look up the pending unit by ID - pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); - - // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed - pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), program); - // add maps to the context for use in processing import - context.getExpUnitContext().setPendingObsUnitByOUId(pendingUnitById); - context.getPendingData().setObservationUnitByNameNoScope(pendingUnitByNameNoScope); - } catch (ApiException e) { + } catch (Exception e) { this.compensate(context, new MiddlewareError(() -> { throw new RuntimeException(e); })); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java index 3f35afea5..8e5a10db7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportPreviewStatistics.java @@ -12,17 +12,14 @@ public class ImportPreviewStatistics extends ExpUnitMiddleware { ExpUnitMiddleware middleware; private Provider newPendingBrAPIObjectsProvider; - private Provider dataValidationProvider; private Provider fieldValidationProvider; @Inject public ImportPreviewStatistics(Provider newPendingBrAPIObjectsProvider, - Provider dataValidationProvider, Provider fieldValidationProvider) { this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link( newPendingBrAPIObjectsProvider.get(), // Construct Pending import objects for new BrAPI data - dataValidationProvider.get(), // Validate data fieldValidationProvider.get()); // Validate fields } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java deleted file mode 100644 index 72fecfd21..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ValidationPrep.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; - -import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; -import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationUnitService; -import tech.tablesaw.api.Table; -import tech.tablesaw.columns.Column; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -import static org.breedinginsight.brapps.importer.services.processors.ObservationProcessor.getObservationHash; - -@Slf4j -public class ValidationPrep extends ExpUnitMiddleware { - ObservationService observationService; - - @Inject - public ValidationPrep(ObservationService observationService) { - this.observationService = observationService; - } - - @Override - public boolean process(ExpUnitMiddlewareContext context) { - // Get all the dynamic columns of the import - ImportUpload upload = context.getImportContext().getUpload(); - Table data = context.getImportContext().getData(); - String[] dynamicColNames = upload.getDynamicColumnNames(); - List> dynamicCols = data.columns(dynamicColNames); - - // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); - Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); - - for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { - Integer rowNum = i; - ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); - - // Fetch the pending import for the row, creating one if null - PendingImport mappedImportRow = context.getImportContext().getMappedBrAPIImport().getOrDefault(rowNum, new PendingImport()); - - List> observations = mappedImportRow.getObservations(); - - String unitId = row.getObsUnitID(); - mappedImportRow.setTrial(context.getExpUnitContext().getPendingTrialByOUId().get(unitId)); - mappedImportRow.setLocation(context.getExpUnitContext().getPendingLocationByOUId().get(unitId)); - mappedImportRow.setStudy(context.getExpUnitContext().getPendingStudyByOUId().get(unitId)); - mappedImportRow.setObservationUnit(context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId)); - mappedImportRow.setGermplasm(context.getExpUnitContext().getPendingGermplasmByOUId().get(unitId)); - - // loop over phenotype column observation data for current row - for (Column column : phenotypeCols) { - String observationHash = observationService.getObservationHash( - pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(unitId).getBrAPIObject().getObservationUnitName(), - getVariableNameFromColumn(column), - pendingStudyByOUId.get(unitId).getBrAPIObject().getStudyName() - ); - - // if value was blank won't be entry in map for this observation - observations.add(observationByHash.get(observationHash)); - } - } - -// try { -// -// } catch (ApiException e) { -// this.compensate(context, new MiddlewareError(() -> { -// throw new RuntimeException(e); -// })); -// } - - return processNext(context); - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index e67ec1b04..6413e6c59 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -1,6 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -16,6 +18,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -35,6 +38,7 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; +import org.jooq.impl.QOM; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -82,11 +86,11 @@ public boolean process(ExpUnitMiddlewareContext context) { List> dynamicCols = data.columns(dynamicColNames); // Collect the columns for observation variable data - List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List> phenotypeCols = dynamicCols.stream().filter(col -> !col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); Set varNames = phenotypeCols.stream().map(Column::name).collect(Collectors.toSet()); // Collect the columns for observation timestamps - List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(ExpImportProcessConstants.TIMESTAMP_PREFIX)).collect(Collectors.toList()); + List> timestampCols = dynamicCols.stream().filter(col -> col.name().startsWith(TIMESTAMP_PREFIX)).collect(Collectors.toList()); Set tsNames = timestampCols.stream().map(Column::name).collect(Collectors.toSet()); // Construct validation errors for any timestamp columns that don't have a matching variable column @@ -212,10 +216,23 @@ public boolean process(ExpUnitMiddlewareContext context) { // Construct a pending observation PendingImportObject pendingPriorObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + // Are overwrites authorized? + boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + String original = null; + // Update the pending phenotypic data if (!cellData.equals(observation.getValue())) { pendingPriorObservation.getBrAPIObject().setValue(cellData); pendingPriorObservation.setState(ImportObjectState.MUTATED); + + // Validation error if user has not chosen to overwrite existing data + if (StringUtils.isNotBlank(cellData) && !canOverwrite) { + ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum, overwriteValErr); + } + + // Record original value in changelog entry + original = observation.getValue(); } // Update the pending timestamp @@ -224,6 +241,38 @@ public boolean process(ExpUnitMiddlewareContext context) { String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); pendingPriorObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); pendingPriorObservation.setState(ImportObjectState.MUTATED); + + // Validation error if user has not chosen to overwrite existing timestamp + if (StringUtils.isNotBlank(cellData) && !canOverwrite) { + ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Timestamp: %s", unitId, tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(rowNum, overwriteValErr); + } + + // Add original timestamp to changelog entry + original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); + + } + + // Record any updates as BrAPI observation additional info + if (context.getImportContext().isCommit()) { + + // Create the changelog field in observation additional info if it does not already exist + if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo().isJsonNull()) { + pendingPriorObservation.getBrAPIObject().setAdditionalInfo(new JsonObject()); + pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo() != null && !pendingPriorObservation.getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + // Construct a changelog entry + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String rightNow = formatter.format(OffsetDateTime.now()); + String reason = Optional.ofNullable(row.getOverwriteReason()).orElse(""); + ChangeLogEntry entry = new ChangeLogEntry(original, reason, context.getImportContext().getUser().getId(), rightNow); + + // Add the entry to the changelog + pendingPriorObservation.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); } // Set the pending prior observation in the pending import for the row From 9ec7de200bdf0f106259eb056eada5babadc0a64 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 20 May 2024 20:33:18 -0400 Subject: [PATCH 58/61] create classes for processed data --- .../AppendOverwritePhenotypesWorkflow.java | 9 +- .../{HandleErr.java => Transaction.java} | 4 +- .../middleware/process/InitialData.java | 81 ++++++++ .../middleware/process/OverwrittenData.java | 177 ++++++++++++++++++ .../middleware/process/UnchangedData.java | 33 ++++ .../process/VisitedObservationData.java | 13 ++ .../process/brapi/PendingObservation.java | 138 ++++++-------- .../service/ObservationService.java | 10 +- .../service/ObservationVariableService.java | 10 +- 9 files changed, 369 insertions(+), 106 deletions(-) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/{HandleErr.java => Transaction.java} (86%) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java index 2d8664968..96878ba62 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java @@ -3,9 +3,8 @@ import io.micronaut.context.annotation.Prototype; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.GetExistingBrAPIData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.HandleErr; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.Transaction; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ValidateAllRowsHaveIDs; -import org.breedinginsight.brapps.importer.services.processors.experiment.middleware.Middleware; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ProcessedData; @@ -19,15 +18,15 @@ public class AppendOverwritePhenotypesWorkflow implements Workflow { ExpUnitMiddleware middleware; - Provider handleErrProvider; + Provider transactionProvider; Provider validateAllRowsHaveIDsProvider; Provider getExistingBrAPIDataProvider; @Inject - public AppendOverwritePhenotypesWorkflow(Provider handleErrProvider, + public AppendOverwritePhenotypesWorkflow(Provider transactionProvider, Provider validateAllRowsHaveIDsProvider, Provider getExistingBrAPIDataProvider) { - this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(handleErrProvider.get(), + this.middleware = (ExpUnitMiddleware) ExpUnitMiddleware.link(transactionProvider.get(), validateAllRowsHaveIDsProvider.get(), getExistingBrAPIDataProvider.get()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java similarity index 86% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java index 267643d63..5e3fd28e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/HandleErr.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/Transaction.java @@ -5,7 +5,9 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @Slf4j -public class HandleErr extends ExpUnitMiddleware { +public class Transaction extends ExpUnitMiddleware { + // TODO: add member for ExpUnitContext + @Override public boolean process(ExpUnitMiddlewareContext context) { return processNext(context); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java new file mode 100644 index 000000000..92f929a8b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -0,0 +1,81 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.services.BrAPIStudyService; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.BRAPI_REFERENCE_SOURCE; + +public class InitialData extends VisitedObservationData { + boolean isCommit; + String cellData; + String phenoColumnName; + ExperimentObservation row; + UUID trialId; + UUID studyId; + String unitId; + String studyYear; + BrAPIObservationUnit observationUnit; + User user; + Program program; + @Inject + StudyService studyService; + @Inject + Gson gson; + + public InitialData(boolean isCommit, + String cellData, + String phenoColumnName, + ExperimentObservation row, + UUID trialId, + UUID studyId, + String unitId, + String studyYear, + BrAPIObservationUnit observationUnit, User user, + Program program) { + this.isCommit = isCommit; + this.cellData = cellData; + this.phenoColumnName = phenoColumnName; + this.row = row; + this.trialId = trialId; + this.studyId = studyId; + this.unitId = unitId; + this.studyYear = studyYear; + this.observationUnit = observationUnit; + this.user = user; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); + + // Generate a new ID for the observation + UUID observationID = UUID.randomUUID(); + + // Construct the new observation + BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, isCommit, program, user, BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + + // Construct a pending observation with a status set to NEW + return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java new file mode 100644 index 000000000..636870243 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -0,0 +1,177 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import io.micronaut.http.HttpStatus; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public class OverwrittenData extends VisitedObservationData { + @Inject + Gson gson; + boolean canOverwrite; + boolean isCommit; + String unitId; + String phenoColumnName; + String timestampColumnName; + String cellData; + String timestamp; + String reason; + BrAPIObservation observation; + UUID userId; + Program program; + + public OverwrittenData(boolean canOverwrite, + boolean isCommit, + String unitId, + String phenoColumnName, + String timestampColumnName, + String cellData, + String timestamp, + String reason, + BrAPIObservation observation, + UUID userId, + Program program) { + this.canOverwrite = canOverwrite; + this.isCommit = isCommit; + this.unitId = unitId; + this.phenoColumnName = phenoColumnName; + this.timestampColumnName = timestampColumnName; + this.cellData = cellData; + this.timestamp = timestamp; + this.reason = reason; + this.observation = observation; + this.userId = userId; + this.program = program; + } + + @Override + public Optional> getValidationErrors() { + List errors = null; + + // Errors for trying to change protected data + if (!canOverwrite) { + errors = new ArrayList<>(); + if (!isValueMatched()) { + errors.add(new ValidationError(phenoColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (!isTimestampMatched()) { + errors.add(new ValidationError(timestampColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, timestampColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + return Optional.ofNullable(errors); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to MUTATED + PendingImportObject pendingUpdatedObservation = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + BrAPIObservation update = pendingUpdatedObservation.getBrAPIObject(); + String original = null; + + if (!isValueMatched()) { + // Update the observation value + update.setValue(cellData); + + // Record original observation value for changelog entry + original = observation.getValue(); + } + + if (!isTimestampMatched()) { + // Update the timestamp + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + + // Add original timestamp to changelog entry + original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); + } + + // If the change is to be committed, attach a record of the change as BrAPI observation additional info + if (isCommit) { + // Create the changelog field in observation additional info if it does not already exist + createAdditionalInfoChangeLog(update); + + // Construct a changelog entry + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String rightNow = formatter.format(OffsetDateTime.now()); + ChangeLogEntry entry = new ChangeLogEntry(original, Optional.ofNullable(reason).orElse(""), userId, rightNow); + + // Add the entry to the changelog + update.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); + } + + return pendingUpdatedObservation; + } + + private void createAdditionalInfoChangeLog(BrAPIObservation update) { + if (update.getAdditionalInfo().isJsonNull()) { + update.setAdditionalInfo(new JsonObject()); + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + if (update.getAdditionalInfo() != null && !update.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + update.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + } + + private boolean isValueMatched() { + return !cellData.equals(observation.getValue()); + } + + private boolean isTimestampMatched() { + if (timestamp == null) { + return observation.getObservationTimeStamp() == null; + } else { + return !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()); + } + } +// private void validateTimeStampValue(String value, +// String columnHeader, ValidationErrors validationErrors, int row) { +// if (StringUtils.isBlank(value)) { +// log.debug(String.format("skipping validation of observation timestamp because there is no value.\n\tvariable: %s\n\trow: %d", columnHeader, row)); +// return; +// } +// if (!validDateValue(value) && !validDateTimeValue(value)) { +// addRowError(columnHeader, "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm", validationErrors, row); +// } +// +// } + + private boolean validDateValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + + private boolean validDateTimeValue(String value) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + try { + formatter.parse(value); + } catch (DateTimeParseException e) { + return false; + } + return true; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java new file mode 100644 index 000000000..8c252b25c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java @@ -0,0 +1,33 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.model.Program; +import org.breedinginsight.utilities.Utilities; + +import java.util.List; +import java.util.Optional; + +public class UnchangedData extends VisitedObservationData { + BrAPIObservation observation; + Program program; + + public UnchangedData(BrAPIObservation observation, Program program) { + this.observation = observation; + this.program = program; + } + @Override + public Optional> getValidationErrors() { + return Optional.empty(); + } + + @Override + public PendingImportObject constructPendingObservation() { + // Construct a pending observation with a status set to EXISTING + PendingImportObject pendingExistingObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); + + return pendingExistingObservation; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java new file mode 100644 index 000000000..b224fc57e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java @@ -0,0 +1,13 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; + +import java.util.List; +import java.util.Optional; + +public abstract class VisitedObservationData { + abstract public Optional> getValidationErrors(); + abstract public PendingImportObject constructPendingObservation(); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index 6413e6c59..eb3161046 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -4,6 +4,8 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.http.HttpStatus; +import io.reactivex.rxjava3.core.Completable; +import io.reactivex.rxjava3.core.Maybe; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -26,6 +28,10 @@ import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.InitialData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.UnchangedData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.VisitedObservationData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; @@ -186,7 +192,7 @@ public boolean process(ExpUnitMiddlewareContext context) { for (Column column : phenotypeCols) { String cellData = column.getString(rowNum); - // Generate hash for looking up prior observation + // Generate hash for looking up prior observation data String studyName = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getStudyName(); String unitName = context.getExpUnitContext().getPendingObsUnitByOUId().get(unitId).getBrAPIObject().getObservationUnitName(); String phenoColumnName = column.name(); @@ -194,8 +200,10 @@ public boolean process(ExpUnitMiddlewareContext context) { // Get timestamp if associated column String timestamp = null; + String tsColumnName = null; if (tsColByPheno.containsKey(phenoColumnName)) { timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + tsColumnName = tsColByPheno.get(phenoColumnName).name(); // If timestamp is not valid, set to midnight if (timestamp != null && !timestamp.isBlank() && (!observationService.validDateTimeValue(timestamp) || !observationService.validDateValue(timestamp))) { @@ -207,105 +215,67 @@ public boolean process(ExpUnitMiddlewareContext context) { } } - // Construct a pending observation if this is a prior observation + VisitedObservationData processedData = null; + // Is there prior observation data for this unit + var? if (observationByObsHash.containsKey(observationHash)) { // Clone the prior observation BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); - // Construct a pending observation - PendingImportObject pendingPriorObservation = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(observation, BrAPIObservation.class, program)); - - // Are overwrites authorized? - boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); - String original = null; - - // Update the pending phenotypic data - if (!cellData.equals(observation.getValue())) { - pendingPriorObservation.getBrAPIObject().setValue(cellData); - pendingPriorObservation.setState(ImportObjectState.MUTATED); - - // Validation error if user has not chosen to overwrite existing data - if (StringUtils.isNotBlank(cellData) && !canOverwrite) { - ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum, overwriteValErr); - } - - // Record original value in changelog entry - original = observation.getValue(); + // Is there a change to the prior data? + if (!cellData.equals(observation.getValue()) || (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()))) { + + // Is prior data protected? + boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); + + // create new instance of OverwrittenData + processedData = new OverwrittenData(canOverwrite, + context.getImportContext().isCommit(), + unitId, + phenoColumnName, + tsColumnName, + cellData, + timestamp, + Optional.ofNullable(row.getOverwriteReason()).orElse(""), + observation, + context.getImportContext().getUser().getId(), + program); + } else { + + // create new instance of UnchangedData + processedData = new UnchangedData(observation, program); } - // Update the pending timestamp - if (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp())) { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timestamp)); - pendingPriorObservation.getBrAPIObject().setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - pendingPriorObservation.setState(ImportObjectState.MUTATED); - - // Validation error if user has not chosen to overwrite existing timestamp - if (StringUtils.isNotBlank(cellData) && !canOverwrite) { - ValidationError overwriteValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Value already exists for ObsUnitId: %s, Timestamp: %s", unitId, tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum, overwriteValErr); - } - - // Add original timestamp to changelog entry - original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); - - } - - // Record any updates as BrAPI observation additional info - if (context.getImportContext().isCommit()) { - - // Create the changelog field in observation additional info if it does not already exist - if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo().isJsonNull()) { - pendingPriorObservation.getBrAPIObject().setAdditionalInfo(new JsonObject()); - pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - if (pendingPriorObservation.getBrAPIObject().getAdditionalInfo() != null && !pendingPriorObservation.getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingPriorObservation.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - // Construct a changelog entry - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String rightNow = formatter.format(OffsetDateTime.now()); - String reason = Optional.ofNullable(row.getOverwriteReason()).orElse(""); - ChangeLogEntry entry = new ChangeLogEntry(original, reason, context.getImportContext().getUser().getId(), rightNow); - - // Add the entry to the changelog - pendingPriorObservation.getBrAPIObject().getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(entry).getAsJsonObject()); - } - - // Set the pending prior observation in the pending import for the row - mappedImportRow.getObservations().add(pendingPriorObservation); - - // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingPriorObservation); - - // Construct a pending observation for new data if no prior observation } else { - UUID trialId = pendingTrial.getId(); - UUID studyId = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(); - String studyYear = context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0); - String seasonDbId = studyService.seasonDbIdToYear(studyYear, program.getId()); // Clone the observation unit BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); - // Generate a new ID for the observation - UUID observationID = UUID.randomUUID(); + // create new instance of InitialData + processedData = new InitialData(context.getImportContext().isCommit(), + cellData, + phenoColumnName, + row, + pendingTrial.getId(), + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), + unitId, + context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getBrAPIObject().getSeasons().get(0), + observationUnit, + context.getImportContext().getUser(), + context.getImportContext().getProgram()); + } - // Construct the new observation - BrAPIObservation newObservation = row.constructBrAPIObservation(cellData, phenoColumnName, seasonDbId, observationUnit, context.getImportContext().isCommit(), program, context.getImportContext().getUser(), BRAPI_REFERENCE_SOURCE, trialId, studyId, UUID.fromString(unitId), observationID); + // Validate processed data + processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); - // Construct a pending observation - PendingImportObject pendingNewObservation = new PendingImportObject<>(ImportObjectState.NEW, newObservation); + // Construct a pending observation + PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); - // Set the new pending observation in the pending import for the row - mappedImportRow.getObservations().add(pendingNewObservation); + // Set the new pending observation in the pending import for the row + mappedImportRow.getObservations().add(pendingProcessedData); - // Add pending observation to map - pendingObservationByHash.put(observationHash, pendingNewObservation); - } + // Add pending observation to map + pendingObservationByHash.put(observationHash, pendingProcessedData); } // Set the pending import for the row diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 67177dad5..38180d402 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -14,6 +14,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; @@ -180,14 +181,7 @@ private void validateObservation(String importHash) { ); // create the changelog field in additional info if it does not already exist - if (pendingObservation.getAdditionalInfo().isJsonNull()) { - pendingObservation.setAdditionalInfo(new JsonObject()); - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } + OverwrittenData.createChangeLog(pendingObservation); // add a new entry to the changelog pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java index bd4ced017..1671d77ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationVariableService.java @@ -14,6 +14,7 @@ import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.ProcessorData; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.ExpUnitContext; import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; @@ -170,14 +171,7 @@ private void validateObservation(String importHash) { ); // create the changelog field in additional info if it does not already exist - if (pendingObservation.getAdditionalInfo().isJsonNull()) { - pendingObservation.setAdditionalInfo(new JsonObject()); - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } - - if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { - pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); - } + OverwrittenData.createChangeLog(pendingObservation); // add a new entry to the changelog pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); From 92af4693b5742cb7b770687734f558f902f7e127 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 11:15:17 -0400 Subject: [PATCH 59/61] create validator classes --- .../validate/field/DateValidator.java | 61 +++++++++++++++++ .../validate/field/FieldValidator.java | 28 ++++++++ .../validate/field/NominalValidator.java | 59 +++++++++++++++++ .../validate/field/NumericalValidator.java | 66 +++++++++++++++++++ .../validate/field/ObservationValidator.java | 12 ++++ .../validate/field/OrdinalValidator.java | 59 +++++++++++++++++ .../service/ObservationService.java | 31 +++++++++ 7 files changed, 316 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java new file mode 100644 index 000000000..2102566ec --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java @@ -0,0 +1,61 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.DATE; + +@Slf4j +@Singleton +public class DateValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public DateValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Is this a timestamp field? + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + + } else { + + // skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // skip if this is not a date trait + if (!DATE.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + if (!observationService.validDateValue(value)) { + return Optional.of(new ValidationError(fieldName, "Incorrect date format detected. Expected YYYY-MM-DD", HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + + + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java new file mode 100644 index 000000000..df7ec338a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; + +@Primary +@Singleton +public class FieldValidator implements ObservationValidator { + private final List validators; + + public FieldValidator(List validators) { + this.validators = validators; + } + + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + return validators.stream() + .map(validator->validator.validateField(fieldName, value, variable)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java new file mode 100644 index 000000000..38bb6c6a3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NOMINAL; + +@Slf4j +@Singleton +public class NominalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public NominalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not an ordinal trait + if (!NOMINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined nominal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java new file mode 100644 index 000000000..cf7165818 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java @@ -0,0 +1,66 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.math.BigDecimal; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.NUMERICAL; + +@Slf4j +@Singleton +public class NumericalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public NumericalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not a numerical trait + if (!NUMERICAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + Optional number = observationService.validNumericValue(value); + Optional validationError = number + .flatMap(num -> { + if (!observationService.validNumericRange(num, variable.getScale())) { + return Optional.of(new ValidationError(fieldName, "Value outside of min/max range detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + return Optional.empty(); + }) + .or(() -> Optional.of(new ValidationError(fieldName, "Non-numeric text detected", HttpStatus.UNPROCESSABLE_ENTITY))); + + return validationError; + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java new file mode 100644 index 000000000..494fcc13c --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java @@ -0,0 +1,12 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.model.Trait; + +import java.util.Optional; + +@FunctionalInterface +public interface ObservationValidator extends Ordered { + Optional validateField(String fieldName, String value, Trait variable); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java new file mode 100644 index 000000000..cd04c56f5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; + +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; +import org.breedinginsight.model.Trait; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.TIMESTAMP_PREFIX; +import static org.breedinginsight.dao.db.enums.DataType.ORDINAL; + +@Slf4j +@Singleton +public class OrdinalValidator implements ObservationValidator { + @Inject + ObservationService observationService; + + public OrdinalValidator(ObservationService observationService) { + this.observationService = observationService; + } + @Override + public Optional validateField(String fieldName, String value, Trait variable) { + if (observationService.isBlankObservation(value)) { + log.debug(String.format("skipping validation of observation because there is no value.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + if (observationService.isNAObservation(value)) { + log.debug(String.format("skipping validation of observation because it is NA.\n\tvariable: %s", fieldName)); + return Optional.empty(); + } + + // Skip if field is a timestamp + if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + return Optional.empty(); + } + + // Skip if there is no trait data + if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { + return Optional.empty(); + } + + // Skip if this is not an ordinal trait + if (!ORDINAL.equals(variable.getScale().getDataType())) { + return Optional.empty(); + } + + // Validate categories + if (!observationService.validCategory(variable.getScale().getCategories(), value)) { + return Optional.of(new ValidationError(fieldName, "Undefined ordinal category detected", HttpStatus.UNPROCESSABLE_ENTITY)); + } + + return Optional.empty(); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index 38180d402..641aa153d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -3,12 +3,14 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import io.micronaut.http.server.exceptions.InternalServerException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -19,10 +21,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Scale; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; import tech.tablesaw.columns.Column; +import java.math.BigDecimal; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; @@ -30,6 +34,33 @@ import java.util.stream.Collectors; public class ObservationService { + public boolean validCategory(List categories, String value) { + Set categoryValues = categories.stream() + .map(category -> category.getValue().toLowerCase()) + .collect(Collectors.toSet()); + return categoryValues.contains(value.toLowerCase()); + } + public boolean validNumericRange(BigDecimal value, Scale validValues) { + // account for empty min or max in valid determination + return (validValues.getValidValueMin() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMin())) >= 0) && + (validValues.getValidValueMax() == null || value.compareTo(BigDecimal.valueOf(validValues.getValidValueMax())) <= 0); + } + public Optional validNumericValue(String value) { + BigDecimal number; + try { + number = new BigDecimal(value); + } catch (NumberFormatException e) { + return Optional.empty(); + } + return Optional.of(number); + } + + public boolean isBlankObservation(String value) { + return StringUtils.isBlank(value); + } + public boolean isNAObservation(String value){ + return value.equalsIgnoreCase("NA"); + } public boolean validDateTimeValue(String value) { DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; try { From 99b17e2ccdd091a45e85bff99336d408ae8128b6 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 16:39:43 -0400 Subject: [PATCH 60/61] use field validator --- .../middleware/process/InitialData.java | 16 ++++- .../middleware/process/OverwrittenData.java | 18 +++++- .../process/brapi/PendingObservation.java | 58 +++++++++++-------- .../validate/field/DateValidator.java | 14 +++-- .../validate/field/ObservationValidator.java | 3 +- 5 files changed, 74 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index 92f929a8b..b048ed87d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -4,16 +4,18 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.api.model.v1.response.ValidationError; -import org.breedinginsight.brapi.v2.services.BrAPIStudyService; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -24,6 +26,7 @@ public class InitialData extends VisitedObservationData { boolean isCommit; String cellData; String phenoColumnName; + Trait trait; ExperimentObservation row; UUID trialId; UUID studyId; @@ -33,6 +36,8 @@ public class InitialData extends VisitedObservationData { User user; Program program; @Inject + FieldValidator fieldValidator; + @Inject StudyService studyService; @Inject Gson gson; @@ -40,6 +45,7 @@ public class InitialData extends VisitedObservationData { public InitialData(boolean isCommit, String cellData, String phenoColumnName, + Trait trait, ExperimentObservation row, UUID trialId, UUID studyId, @@ -50,6 +56,7 @@ public InitialData(boolean isCommit, this.isCommit = isCommit; this.cellData = cellData; this.phenoColumnName = phenoColumnName; + this.trait = trait; this.row = row; this.trialId = trialId; this.studyId = studyId; @@ -61,7 +68,12 @@ public InitialData(boolean isCommit, } @Override public Optional> getValidationErrors() { - return Optional.empty(); + List errors = new ArrayList<>(); + + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 636870243..40e3f72f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -10,7 +10,9 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; @@ -23,11 +25,14 @@ import java.util.UUID; public class OverwrittenData extends VisitedObservationData { + @Inject + FieldValidator fieldValidator; @Inject Gson gson; boolean canOverwrite; boolean isCommit; String unitId; + Trait trait; String phenoColumnName; String timestampColumnName; String cellData; @@ -40,6 +45,7 @@ public class OverwrittenData extends VisitedObservationData { public OverwrittenData(boolean canOverwrite, boolean isCommit, String unitId, + Trait trait, String phenoColumnName, String timestampColumnName, String cellData, @@ -51,6 +57,7 @@ public OverwrittenData(boolean canOverwrite, this.canOverwrite = canOverwrite; this.isCommit = isCommit; this.unitId = unitId; + this.trait = trait; this.phenoColumnName = phenoColumnName; this.timestampColumnName = timestampColumnName; this.cellData = cellData; @@ -63,11 +70,10 @@ public OverwrittenData(boolean canOverwrite, @Override public Optional> getValidationErrors() { - List errors = null; + List errors = new ArrayList<>(); // Errors for trying to change protected data if (!canOverwrite) { - errors = new ArrayList<>(); if (!isValueMatched()) { errors.add(new ValidationError(phenoColumnName, String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", unitId, phenoColumnName), HttpStatus.UNPROCESSABLE_ENTITY)); } @@ -76,7 +82,13 @@ public Optional> getValidationErrors() { } } - return Optional.ofNullable(errors); + // Validate observation value + fieldValidator.validateField(phenoColumnName, cellData, trait).ifPresent(errors::add); + + // Validate timestamp + fieldValidator.validateField(timestampColumnName, timestamp, null).ifPresent(errors::add); + + return Optional.ofNullable(errors.isEmpty() ? null : errors); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index eb3161046..bba8f886c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -1,12 +1,8 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.brapi; import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import io.micronaut.http.HttpStatus; -import io.reactivex.rxjava3.core.Completable; -import io.reactivex.rxjava3.core.Maybe; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; @@ -20,7 +16,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; @@ -32,7 +27,7 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.UnchangedData; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.VisitedObservationData; -import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; @@ -44,13 +39,11 @@ import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.Utilities; -import org.jooq.impl.QOM; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; import javax.inject.Inject; import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -65,6 +58,7 @@ public class PendingObservation extends ExpUnitMiddleware { BrAPIObservationDAO brAPIObservationDAO; FileMappingUtil fileMappingUtil; Gson gson; + FieldValidator fieldValidator; @Inject public PendingObservation(StudyService studyService, @@ -72,7 +66,8 @@ public PendingObservation(StudyService studyService, BrAPIObservationDAO brAPIObservationDAO, ObservationService observationService, FileMappingUtil fileMappingUtil, - Gson gson) { + Gson gson, + FieldValidator fieldValidator) { this.studyService = studyService; this.observationVariableService = observationVariableService; this.brAPIObservationDAO = brAPIObservationDAO; @@ -126,6 +121,16 @@ public boolean process(ExpUnitMiddlewareContext context) { Program program = context.getImportContext().getProgram(); List traits = observationVariableService.fetchTraitsByName(varNames, program); + // Map trait by phenotype column name + Map traitByPhenoColName = traits.stream().collect( + Collectors.toMap( + trait -> trait.getObservationVariableName().toUpperCase(), // Use uppercase keys for case-insensitivity + trait -> trait, + (trait1, trait2) -> trait1, // Merge function + CaseInsensitiveMap::new // Supplier for creating a CaseInsensitiveMap + ) + ); + // Sort the traits to match the order of the headers in the import file List sortedTraits = fileMappingUtil.sortByField(List.copyOf(varNames), new ArrayList<>(traits), TraitEntity::getObservationVariableName); @@ -173,8 +178,7 @@ public boolean process(ExpUnitMiddlewareContext context) { // Build new pending observation data for each phenotype Map> pendingObservationByHash = new HashMap<>(); - - // Checking all import rows for data + // Process observation data for each row for (int i = 0; i < context.getImportContext().getImportRows().size(); i++) { Integer rowNum = i; ExperimentObservation row = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum); @@ -199,20 +203,20 @@ public boolean process(ExpUnitMiddlewareContext context) { String observationHash = observationService.getObservationHash(unitName, phenoColumnName, studyName); // Get timestamp if associated column - String timestamp = null; + var cell = new Object() { // mutable reference object to make timestamp accessible in anonymous methods + String timestamp = null; + }; String tsColumnName = null; if (tsColByPheno.containsKey(phenoColumnName)) { - timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); + cell.timestamp = tsColByPheno.get(phenoColumnName).getString(rowNum); tsColumnName = tsColByPheno.get(phenoColumnName).name(); // If timestamp is not valid, set to midnight - if (timestamp != null && !timestamp.isBlank() && (!observationService.validDateTimeValue(timestamp) || !observationService.validDateValue(timestamp))) { - timestamp += MIDNIGHT; + fieldValidator.validateField(tsColumnName, cell.timestamp, null).ifPresent(err->{ + cell.timestamp += MIDNIGHT; + validationErrors.addError(rowNum, err); + }); - // Add a validation error - ValidationError timestampValErr = new ValidationError(tsColByPheno.get(phenoColumnName).name(), String.format("Timestamp format is not valid for %s", tsColByPheno.get(phenoColumnName).name()), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(rowNum, timestampValErr); - } } VisitedObservationData processedData = null; @@ -223,19 +227,23 @@ public boolean process(ExpUnitMiddlewareContext context) { BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if (!cellData.equals(observation.getValue()) || (timestamp != null && !OffsetDateTime.parse(timestamp).equals(observation.getObservationTimeStamp()))) { + if (!cellData.equals(observation.getValue()) || (cell.timestamp != null && !OffsetDateTime.parse(cell.timestamp).equals(observation.getObservationTimeStamp()))) { // Is prior data protected? boolean canOverwrite = context.getImportContext().isCommit() && "false".equals( row.getOverwrite() == null ? "false" : row.getOverwrite()); - // create new instance of OverwrittenData + // Clone the trait + Trait changeTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); + + // Create new instance of OverwrittenData processedData = new OverwrittenData(canOverwrite, context.getImportContext().isCommit(), unitId, + changeTrait, phenoColumnName, tsColumnName, cellData, - timestamp, + cell.timestamp, Optional.ofNullable(row.getOverwriteReason()).orElse(""), observation, context.getImportContext().getUser().getId(), @@ -248,13 +256,15 @@ public boolean process(ExpUnitMiddlewareContext context) { } else { - // Clone the observation unit + // Clone the observation unit and trait BrAPIObservationUnit observationUnit = gson.fromJson(gson.toJson(context.getExpUnitContext().getPendingObsUnitByOUId().get(row.getExpUnitId()).getBrAPIObject()), BrAPIObservationUnit.class); + Trait initialTrait = gson.fromJson(gson.toJson(traitByPhenoColName.get(phenoColumnName)), Trait.class); // create new instance of InitialData processedData = new InitialData(context.getImportContext().isCommit(), cellData, phenoColumnName, + initialTrait, row, pendingTrial.getId(), context.getExpUnitContext().getPendingStudyByOUId().get(unitId).getId(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java index 2102566ec..d10c7c865 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java @@ -18,6 +18,8 @@ public class DateValidator implements ObservationValidator { @Inject ObservationService observationService; + private final String dateMessage = "Incorrect date format detected. Expected YYYY-MM-DD"; + private final String dateTimeMessage = "Incorrect datetime format detected. Expected YYYY-MM-DD or YYYY-MM-DDThh:mm:ss+hh:mm"; public DateValidator(ObservationService observationService) { this.observationService = observationService; @@ -36,26 +38,28 @@ public Optional validateField(String fieldName, String value, T // Is this a timestamp field? if (fieldName.startsWith(TIMESTAMP_PREFIX)) { + if (!observationService.validDateValue(value) || !observationService.validDateTimeValue(value)) { + return Optional.of(new ValidationError(fieldName, dateTimeMessage, HttpStatus.UNPROCESSABLE_ENTITY)); + } } else { - // skip if there is no trait data + // Skip if there is no trait data if (variable == null || variable.getScale() == null || variable.getScale().getDataType() == null) { return Optional.empty(); } - // skip if this is not a date trait + // Skip if this is not a date trait if (!DATE.equals(variable.getScale().getDataType())) { return Optional.empty(); } + // Validate date if (!observationService.validDateValue(value)) { - return Optional.of(new ValidationError(fieldName, "Incorrect date format detected. Expected YYYY-MM-DD", HttpStatus.UNPROCESSABLE_ENTITY)); + return Optional.of(new ValidationError(fieldName, dateMessage, HttpStatus.UNPROCESSABLE_ENTITY)); } } - - return Optional.empty(); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java index 494fcc13c..825913732 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.model.Trait; @@ -8,5 +9,5 @@ @FunctionalInterface public interface ObservationValidator extends Ordered { - Optional validateField(String fieldName, String value, Trait variable); + Optional validateField(@NonNull String fieldName, @NonNull String value, Trait variable); } From 9141e9b960cdbb269bbc179e6b255a5e52e5abb7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 May 2024 20:48:13 -0400 Subject: [PATCH 61/61] create preview statistics class --- .../middleware/process/AppendStatistic.java | 87 +++++++++++++++++++ .../middleware/process/InitialData.java | 7 +- .../middleware/process/OverwrittenData.java | 7 +- .../middleware/process/UnchangedData.java | 5 ++ .../process/VisitedObservationData.java | 1 + .../process/brapi/PendingObservation.java | 20 +++-- .../validate/{field => }/DateValidator.java | 2 +- .../validate/{field => }/FieldValidator.java | 2 +- .../{field => }/NominalValidator.java | 2 +- .../{field => }/NumericalValidator.java | 2 +- .../{field => }/ObservationValidator.java | 2 +- .../{field => }/OrdinalValidator.java | 2 +- 12 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/DateValidator.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/FieldValidator.java (94%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/NominalValidator.java (97%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/NumericalValidator.java (98%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/ObservationValidator.java (89%) rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/{field => }/OrdinalValidator.java (97%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java new file mode 100644 index 000000000..aeb88cfe8 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/AppendStatistic.java @@ -0,0 +1,87 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process; + +import io.micronaut.context.annotation.Prototype; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; + +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; + +@Prototype +public class AppendStatistic { + private final HashSet environmentNames; + private final HashSet observationUnitIds; + private final HashSet gids; + private int newCount; + private int existingCount; + private int mutatedCount; + + public AppendStatistic() { + this.environmentNames = new HashSet<>(); + this.observationUnitIds = new HashSet<>(); + this.gids = new HashSet<>(); + this.newCount = 0; + this.existingCount = 0; + this.mutatedCount = 0; + } + + public int incrementNewCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.newCount += increment; + + return this.newCount; + } + public int incrementExistingCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.existingCount += increment; + + return this.existingCount; + } + public int incrementMutatedCount(Integer value) { + int increment = 0; + if (value == null) { + increment = 1; + } else if (value >= 0) { + increment = value; + } + this.mutatedCount += increment; + + return this.mutatedCount; + } + public void addEnvironmentName(String name) { + Optional.ofNullable(name).ifPresent(environmentNames::add); + } + public void addObservationUnitId(String id) { + Optional.ofNullable(id).ifPresent(observationUnitIds::add); + } + public void addGid(String gid) { + Optional.ofNullable(gid).ifPresent(gids::add); + } + public Map constructPreviewMap() { + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder().newObjectCount(environmentNames.size()).build(); + ImportPreviewStatistics observationUnitsStats = ImportPreviewStatistics.builder().newObjectCount(observationUnitIds.size()).build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder().newObjectCount(gids.size()).build(); + ImportPreviewStatistics newStats = ImportPreviewStatistics.builder().newObjectCount(newCount).build(); + ImportPreviewStatistics existingStats = ImportPreviewStatistics.builder().newObjectCount(existingCount).build(); + ImportPreviewStatistics mutatedStats = ImportPreviewStatistics.builder().newObjectCount(mutatedCount).build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", observationUnitsStats, + "GIDs", gidStats, + "Observations", newStats, + "Existing_Observations", existingStats, + "Mutated_Observations", mutatedStats + ); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java index b048ed87d..48dd9924f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/InitialData.java @@ -7,7 +7,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.service.StudyService; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; @@ -90,4 +90,9 @@ public PendingImportObject constructPendingObservation() { return new PendingImportObject<>(ImportObjectState.NEW, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementNewCount(1); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java index 40e3f72f5..a6d7c23ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/OverwrittenData.java @@ -10,7 +10,7 @@ import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.utilities.Utilities; @@ -133,6 +133,11 @@ public PendingImportObject constructPendingObservation() { return pendingUpdatedObservation; } + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementMutatedCount(1); + } + private void createAdditionalInfoChangeLog(BrAPIObservation update) { if (update.getAdditionalInfo().isJsonNull()) { update.setAdditionalInfo(new JsonObject()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java index 8c252b25c..6c3245966 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/UnchangedData.java @@ -30,4 +30,9 @@ public PendingImportObject constructPendingObservation() { return pendingExistingObservation; } + + @Override + public void updateTally(AppendStatistic statistic) { + statistic.incrementExistingCount(1); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java index b224fc57e..bdd064b93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/VisitedObservationData.java @@ -10,4 +10,5 @@ public abstract class VisitedObservationData { abstract public Optional> getValidationErrors(); abstract public PendingImportObject constructPendingObservation(); + abstract public void updateTally(AppendStatistic statistic); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java index bba8f886c..af82901a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/brapi/PendingObservation.java @@ -23,11 +23,8 @@ import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.ExpUnitMiddleware; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.InitialData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.OverwrittenData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.UnchangedData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.VisitedObservationData; -import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field.FieldValidator; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.*; +import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.FieldValidator; import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpUnitMiddlewareContext; import org.breedinginsight.brapps.importer.services.processors.experiment.model.MiddlewareError; import org.breedinginsight.brapps.importer.services.processors.experiment.service.ObservationService; @@ -59,6 +56,7 @@ public class PendingObservation extends ExpUnitMiddleware { FileMappingUtil fileMappingUtil; Gson gson; FieldValidator fieldValidator; + AppendStatistic statistic; @Inject public PendingObservation(StudyService studyService, @@ -67,13 +65,15 @@ public PendingObservation(StudyService studyService, ObservationService observationService, FileMappingUtil fileMappingUtil, Gson gson, - FieldValidator fieldValidator) { + FieldValidator fieldValidator, + AppendStatistic statistic) { this.studyService = studyService; this.observationVariableService = observationVariableService; this.brAPIObservationDAO = brAPIObservationDAO; this.observationService = observationService; this.fileMappingUtil = fileMappingUtil; this.gson = gson; + this.statistic = statistic; } @Override @@ -278,6 +278,14 @@ public boolean process(ExpUnitMiddlewareContext context) { // Validate processed data processedData.getValidationErrors().ifPresent(errList -> errList.forEach(e->validationErrors.addError(rowNum, e))); + // Update import preview statistics + processedData.updateTally(statistic); + statistic.addEnvironmentName(studyName); + // TODO: change null values to actual data + // TODO: change signature to take two args, studyName and unitName + statistic.addObservationUnitId(null); + statistic.addGid(null); + // Construct a pending observation PendingImportObject pendingProcessedData = processedData.constructPendingObservation(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java index d10c7c865..615da28ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/DateValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/DateValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java similarity index 94% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java index df7ec338a..724aa01d3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/FieldValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/FieldValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.context.annotation.Primary; import org.breedinginsight.api.model.v1.response.ValidationError; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java index 38bb6c6a3..7f2cf50df 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NominalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NominalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java index cf7165818..d90d8d115 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/NumericalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/NumericalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java similarity index 89% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java index 825913732..e378cd228 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/ObservationValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/ObservationValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.core.annotation.NonNull; import io.micronaut.core.order.Ordered; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java index cd04c56f5..76fbcab04 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/field/OrdinalValidator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/validate/OrdinalValidator.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate.field; +package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.validate; import io.micronaut.http.HttpStatus; import lombok.extern.slf4j.Slf4j;